diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 00000000000..96781221cf1 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,362 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Go + +on: + push: + paths: + - '.github/workflows/go.yml' + - 'ci/docker/*_go.dockerfile' + - 'ci/scripts/go_*' + - 'go/**' + pull_request: + paths: + - '.github/workflows/go.yml' + - 'ci/docker/*_go.dockerfile' + - 'ci/docker/**' + - 'ci/scripts/go_*' + - 'go/**' + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + + docker: + name: AMD64 Debian 11 Go ${{ matrix.go }} + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + go: [1.17, 1.18] + include: + - go: 1.17 + staticcheck: v0.2.2 + - go: 1.18 + staticcheck: latest + env: + GO: ${{ matrix.go }} + STATICCHECK: ${{ matrix.staticcheck }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + run: archery docker run debian-go + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + continue-on-error: true + run: archery docker push debian-go + - name: Run Benchmarks + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + env: + CONBENCH_URL: https://conbench.ursa.dev + CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }} + CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }} + CONBENCH_REF: ${{ github.ref_name }} + run: | + pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python + python ci/scripts/go_bench_adapt.py + + docker_cgo: + name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + go: [1.17, 1.18] + include: + - go: 1.17 + staticcheck: v0.2.2 + - go: 1.18 + staticcheck: latest + env: + GO: ${{ matrix.go }} + STATICCHECK: ${{ matrix.staticcheck }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + run: archery docker run debian-go-cgo + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + continue-on-error: true + run: archery docker push debian-go-cgo + + + docker_cgo_python: + name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO Python + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + go: [1.17, 1.18] + include: + - go: 1.17 + staticcheck: v0.2.2 + - go: 1.18 + staticcheck: latest + env: + GO: ${{ matrix.go }} + STATICCHECK: ${{ matrix.staticcheck }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + run: archery docker run debian-go-cgo-python + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + continue-on-error: true + run: archery docker push debian-go-cgo-python + + windows: + name: AMD64 Windows 2019 Go ${{ matrix.go }} + runs-on: windows-2019 + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + go: [1.17, 1.18] + include: + - go: 1.17 + staticcheck: v0.2.2 + - go: 1.18 + staticcheck: latest + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + - name: Install go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go }} + cache: true + cache-dependency-path: go/go.sum + - name: Install staticcheck + run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }} + - name: Build + shell: bash + run: ci/scripts/go_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/go_test.sh $(pwd) + + macos: + name: AMD64 macOS 11 Go ${{ matrix.go }} + runs-on: macos-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + go: [1.17, 1.18] + include: + - go: 1.17 + staticcheck: v0.2.2 + - go: 1.18 + staticcheck: latest + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + - name: Install go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go }} + cache: true + cache-dependency-path: go/go.sum + - name: Install staticcheck + run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }} + - name: Build + shell: bash + run: ci/scripts/go_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/go_test.sh $(pwd) + - name: Setup Python + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Run Benchmarks + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + shell: bash + env: + CONBENCH_URL: 'https://conbench.ursa.dev' + CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }} + CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }} + CONBENCH_REF: ${{ github.ref_name }} + run: | + pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python + python ci/scripts/go_bench_adapt.py + + + macos-cgo: + name: AMD64 macOS 11 Go ${{ matrix.go }} - CGO + runs-on: macos-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + go: [1.17, 1.18] + include: + - go: 1.17 + staticcheck: v0.2.2 + - go: 1.18 + staticcheck: latest + env: + ARROW_GO_TESTCGO: "1" + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + - name: Install go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go }} + cache: true + cache-dependency-path: go/go.sum + - name: Brew Install Arrow + shell: bash + run: brew install apache-arrow + - name: Install staticcheck + run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }} + - name: Build + shell: bash + run: ci/scripts/go_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/go_test.sh $(pwd) + + windows-mingw: + name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} CGO + runs-on: windows-2019 + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + mingw-n-bits: + #- 32 runtime handling for CGO needs 64-bit currently + - 64 + env: + ARROW_GO_TESTCGO: "1" + MINGW_LINT: "1" + steps: + - name: Disable Crash Dialogs + run: | + reg add ` + "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` + /v DontShowUI ` + /t REG_DWORD ` + /d 1 ` + /f + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + - uses: msys2/setup-msys2@v2 + with: + msystem: MINGW${{ matrix.mingw-n-bits }} + update: true + - name: Setup MSYS2 + shell: msys2 {0} + run: | + ci/scripts/msys2_setup.sh cgo + - name: Update CGO Env vars + shell: msys2 {0} + run: | + echo "CGO_CPPFLAGS=-I$(cygpath --windows ${MINGW_PREFIX}/include)" >> $GITHUB_ENV + echo "CGO_LDFLAGS=-g -O2 -L$(cygpath --windows ${MINGW_PREFIX}/lib) -L$(cygpath --windows ${MINGW_PREFIX}/bin)" >> $GITHUB_ENV + echo "MINGW_PREFIX=$(cygpath --windows ${MINGW_PREFIX})" >> $GITHUB_ENV + - name: Install go + uses: actions/setup-go@v3 + with: + go-version: '1.18' + cache: true + cache-dependency-path: go/go.sum + - name: Install staticcheck + run: go install honnef.co/go/tools/cmd/staticcheck@latest + - name: Build + shell: bash + run: ci/scripts/go_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/go_test.sh $(pwd) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml new file mode 100644 index 00000000000..9173f0e530b --- /dev/null +++ b/.github/workflows/r.yml @@ -0,0 +1,334 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: R + +on: + push: + paths: + - ".github/workflows/r.yml" + - "ci/scripts/r_*.sh" + - "ci/scripts/cpp_*.sh" + - "ci/scripts/PKGBUILD" + - "ci/etc/rprofile" + - "ci/docker/**" + - "cpp/**" + - "r/**" + pull_request: + paths: + - ".github/workflows/r.yml" + - "ci/scripts/r_*.sh" + - "ci/scripts/cpp_*.sh" + - "ci/scripts/PKGBUILD" + - "ci/etc/rprofile" + - "ci/docker/**" + - "cpp/**" + - "r/**" + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + +jobs: + ubuntu: + name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }} Force-Tests ${{ matrix.force-tests }} + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + r: ["4.2"] + ubuntu: [20.04] + force-tests: ["true"] + env: + R: ${{ matrix.r }} + UBUNTU: ${{ matrix.ubuntu }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Cache Docker Volumes + uses: actions/cache@v3 + with: + path: .docker + # As this key is identical on both matrix builds only one will be able to successfully cache, + # this is fine as there are no differences in the build + key: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} + restore-keys: | + ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- + ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}- + - name: Check pkgdown reference sections + run: ci/scripts/r_pkgdown_check.sh + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + # Setting a non-default and non-probable Marquesas French Polynesia time + # it has both with a .45 offset and very very few people who live there. + archery docker run -e TZ=MART -e ARROW_R_FORCE_TESTS=${{ matrix.force-tests }} ubuntu-r + - name: Dump install logs + run: cat r/check/arrow.Rcheck/00install.out + if: always() + - name: Dump test logs + run: cat r/check/arrow.Rcheck/tests/testthat.Rout* + if: always() + - name: Save the test output + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-output + path: r/check/arrow.Rcheck/tests/testthat.Rout* + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + continue-on-error: true + run: archery docker push ubuntu-r + + bundled: + name: "${{ matrix.config.org }}/${{ matrix.config.image }}:${{ matrix.config.tag }}" + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + config: + - { org: "rhub", image: "debian-gcc-devel", tag: "latest", devtoolset: "" } + env: + R_ORG: ${{ matrix.config.org }} + R_IMAGE: ${{ matrix.config.image }} + R_TAG: ${{ matrix.config.tag }} + DEVTOOLSET_VERSION: ${{ matrix.config.devtoolset }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + # Don't set a TZ here to test that case. These builds will have the following warning in them: + # System has not been booted with systemd as init system (PID 1). Can't operate. + # Failed to connect to bus: Host is down + archery docker run -e TZ="" r + - name: Dump install logs + run: cat r/check/arrow.Rcheck/00install.out + if: always() + - name: Dump test logs + run: cat r/check/arrow.Rcheck/tests/testthat.Rout* + if: always() + - name: Save the test output + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-output + path: r/check/arrow.Rcheck/tests/testthat.Rout* + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + continue-on-error: true + run: archery docker push r + + windows-cpp: + name: AMD64 Windows C++ RTools ${{ matrix.config.rtools }} ${{ matrix.config.arch }} + runs-on: windows-2019 + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + config: + - { rtools: 40, arch: 'ucrt64' } + steps: + - run: git config --global core.autocrlf false + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Setup ccache + shell: bash + run: | + ci/scripts/ccache_setup.sh + echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV + - name: Cache ccache + uses: actions/cache@v3 + with: + path: ccache + key: r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-${{ github.run_id }} + restore-keys: | + r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}- + r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}- + - uses: r-lib/actions/setup-r@v2 + with: + r-version: "4.1" + rtools-version: 40 + Ncpus: 2 + - name: Build Arrow C++ + shell: bash + env: + MINGW_ARCH: ${{ matrix.config.arch }} + run: ci/scripts/r_windows_build.sh + - name: Rename libarrow.zip + # So that they're unique when multiple are downloaded in the next step + shell: bash + run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip + - uses: actions/upload-artifact@v3 + with: + name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip + path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip + + windows-r: + needs: [windows-cpp] + name: AMD64 Windows R ${{ matrix.config.rversion }} RTools ${{ matrix.config.rtools }} + runs-on: windows-2019 + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + config: + - { rtools: 42, rversion: "4.2" } + - { rtools: 42, rversion: "devel" } + env: + ARROW_R_CXXFLAGS: "-Werror" + _R_CHECK_TESTS_NLINES_: 0 + steps: + - run: git config --global core.autocrlf false + - name: Checkout Arrow + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - run: mkdir r/windows + - name: Download artifacts + if: ${{ matrix.config.rtools == 42 }} + uses: actions/download-artifact@v3 + with: + name: libarrow-rtools40-ucrt64.zip + path: r/windows + - name: Unzip and rezip libarrows + shell: bash + run: | + cd r/windows + ls *.zip | xargs -n 1 unzip -uo + rm -rf *.zip + - uses: r-lib/actions/setup-r@v2 + with: + r-version: ${{ matrix.config.rversion }} + rtools-version: ${{ matrix.config.rtools }} + Ncpus: 2 + - uses: r-lib/actions/setup-r-dependencies@v2 + env: + GITHUB_PAT: "${{ github.token }}" + with: + # For some arcane reason caching does not work on the windows runners + # most likely due to https://github.com/actions/cache/issues/815 + cache: false + working-directory: 'r' + extra-packages: | + any::rcmdcheck + - name: Install MinIO + shell: bash + run: | + mkdir -p "$HOME/.local/bin" + curl \ + --output "$HOME/.local/bin/minio.exe" \ + https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2022-05-26T05-48-41Z + chmod +x "$HOME/.local/bin/minio.exe" + echo "$HOME/.local/bin" >> $GITHUB_PATH + # TODO(ARROW-17149): figure out why the GCS tests are hanging on Windows + # - name: Install Google Cloud Storage Testbench + # shell: bash + # run: ci/scripts/install_gcs_testbench.sh default + - name: Check + shell: Rscript {0} + run: | + # Because we do R CMD build and r/windows is in .Rbuildignore, + # assemble the libarrow.zip file and pass it as an env var + setwd("r/windows") + zip("libarrow.zip", ".") + setwd("..") + + Sys.setenv( + RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), + MAKEFLAGS = paste0("-j", parallel::detectCores()), + ARROW_R_DEV = TRUE, + "_R_CHECK_FORCE_SUGGESTS_" = FALSE + ) + rcmdcheck::rcmdcheck(".", + build_args = '--no-build-vignettes', + args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'), + error_on = 'warning', + check_dir = 'check', + timeout = 3600 + ) + - name: Run lintr + if: ${{ matrix.config.rversion == '4.2' }} + env: + NOT_CRAN: "true" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: Rscript {0} + working-directory: r + run: | + Sys.setenv( + RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), + MAKEFLAGS = paste0("-j", parallel::detectCores()), + ARROW_R_DEV = TRUE, + "_R_CHECK_FORCE_SUGGESTS_" = FALSE + ) + # we use pak for package installation since it is faster, safer and more convenient + pak::local_install() + pak::pak("lintr") + lintr::expect_lint_free() + - name: Dump install logs + shell: cmd + run: cat r/check/arrow.Rcheck/00install.out + if: always() + - name: Dump test logs + shell: bash + run: find r/check -name 'testthat.Rout*' -exec cat '{}' \; || true + if: always() diff --git a/c_glib/test/test-orc-file-reader.rb b/c_glib/test/test-orc-file-reader.rb index 38900cf12f3..6626c67c3ab 100644 --- a/c_glib/test/test-orc-file-reader.rb +++ b/c_glib/test/test-orc-file-reader.rb @@ -185,8 +185,8 @@ def all_columns test("select fields") do require_gi_bindings(3, 2, 6) @reader.field_indices = [1, 3] - assert_equal(build_table("boolean1" => build_boolean_array([false, true]), - "short1" => build_int16_array([1024, 2048])), + assert_equal(build_table("byte1" => build_int8_array([1, 100]), + "int1" => build_int32_array([65536, 65536])), @reader.read_stripes) end end @@ -200,10 +200,8 @@ def all_columns test("select fields") do require_gi_bindings(3, 2, 6) @reader.field_indices = [1, 3] - boolean1 = build_boolean_array([false, true]) - short1 = build_int16_array([1024, 2048]) - assert_equal(build_record_batch("boolean1" => boolean1, - "short1" => short1), + assert_equal(build_record_batch("byte1" => build_int8_array([1, 100]), + "int1" => build_int32_array([65536, 65536])), @reader.read_stripe(0)) end end diff --git a/ci/conan/all/conanfile.py b/ci/conan/all/conanfile.py index 26abcc028b3..726f83239b5 100644 --- a/ci/conan/all/conanfile.py +++ b/ci/conan/all/conanfile.py @@ -302,7 +302,7 @@ def requirements(self): if self._with_thrift(): self.requires("thrift/0.16.0") if self._with_protobuf(): - self.requires("protobuf/3.21.1") + self.requires("protobuf/3.21.4") if self._with_jemalloc(): self.requires("jemalloc/5.2.1") if self._with_boost(): @@ -346,7 +346,7 @@ def requirements(self): if self.options.with_zstd: self.requires("zstd/1.5.2") if self._with_re2(): - self.requires("re2/20220201") + self.requires("re2/20220601") if self._with_utf8proc(): self.requires("utf8proc/2.7.0") if self.options.with_backtrace: diff --git a/ci/scripts/go_bench.sh b/ci/scripts/go_bench.sh new file mode 100644 index 00000000000..5347b42524e --- /dev/null +++ b/ci/scripts/go_bench.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# this will output the benchmarks to STDOUT but if `-json` is passed +# as the second argument, it will create a file "bench_stats.json" +# in the directory this is called from containing a json representation + +set -ex + +# simplistic semver comparison +verlte() { + [ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ] +} +verlt() { + [ "$1" = "$2" ] && return 1 || verlte $1 $2 +} + +ver=`go env GOVERSION` + +source_dir=${1}/go + +export PARQUET_TEST_DATA=${1}/cpp/submodules/parquet-testing/data +pushd ${source_dir} + +go test -bench=. -benchmem -run=^$ ./... | tee bench_stat.dat + +if verlte "1.18" "${ver#go}"; then + go test -bench=. -benchmem -run=^$ ./arrow/compute | tee bench_stat_compute.dat +fi + +popd + +if [[ "$2" = "-json" ]]; then + go install go.bobheadxi.dev/gobenchdata@latest + export PATH=`go env GOPATH`/bin:$PATH + cat ${source_dir}/bench_*.dat | gobenchdata --json bench_stats.json +fi + +rm ${source_dir}/bench_*.dat \ No newline at end of file diff --git a/ci/scripts/go_bench_adapt.py b/ci/scripts/go_bench_adapt.py new file mode 100644 index 00000000000..db1c09cbc59 --- /dev/null +++ b/ci/scripts/go_bench_adapt.py @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import os +import uuid +import logging +from pathlib import Path +from typing import List + +from benchadapt import BenchmarkResult +from benchadapt.adapters import BenchmarkAdapter +from benchadapt.log import log + +log.setLevel(logging.DEBUG) + +ARROW_ROOT = Path(__file__).parent.parent.parent.resolve() +SCRIPTS_PATH = ARROW_ROOT / "ci" / "scripts" +RUN_REASON = "commit" if os.environ.get("CONBENCH_REF") == "master" else "branch" + +class GoAdapter(BenchmarkAdapter): + result_file = "bench_stats.json" + command = ["bash", SCRIPTS_PATH / "go_bench.sh", ARROW_ROOT, "-json"] + + def __init__(self, *args, **kwargs) -> None: + super().__init__(command=self.command, *args, **kwargs) + + def _transform_results(self) -> List[BenchmarkResult]: + with open(self.result_file, "r") as f: + raw_results = json.load(f) + + run_id = uuid.uuid4().hex + parsed_results = [] + for suite in raw_results[0]["Suites"]: + batch_id = uuid.uuid4().hex + pkg = suite["Pkg"] + + for benchmark in suite["Benchmarks"]: + data = benchmark["Mem"]["MBPerSec"] * 1e6 + time = 1 / benchmark["NsPerOp"] * 1e9 + + name = benchmark["Name"].removeprefix('Benchmark') + ncpu = name[name.rfind('-')+1:] + pieces = name[:-(len(ncpu)+1)].split('/') + + parsed = BenchmarkResult( + run_id=run_id, + batch_id=batch_id, + stats={ + "data": [data], + "unit": "b/s", + "times": [time], + "time_unit": "i/s", + "iterations": benchmark["Runs"], + }, + context={ + "benchmark_language": "Go", + "goos": suite["Goos"], + "goarch": suite["Goarch"], + }, + tags={ + "pkg": pkg, + "num_cpu": ncpu, + "name": pieces[0], + "params": '/'.join(pieces[1:]), + }, + run_reason=RUN_REASON, + ) + parsed.run_name = f"{parsed.run_reason}: {parsed.github['commit']}" + parsed_results.append(parsed) + + return parsed_results + + +if __name__ == "__main__": + go_adapter = GoAdapter(result_fields_override={"info":{}}) + go_adapter() \ No newline at end of file diff --git a/ci/scripts/go_test.sh b/ci/scripts/go_test.sh index e31fa555642..54b05c3cc2b 100755 --- a/ci/scripts/go_test.sh +++ b/ci/scripts/go_test.sh @@ -61,7 +61,7 @@ pushd ${source_dir}/arrow TAGS="assert,test" if [[ -n "${ARROW_GO_TESTCGO}" ]]; then if [[ "${MSYSTEM}" = "MINGW64" ]]; then - export PATH=${MINGW_PREFIX}/bin:$PATH + export PATH=${MINGW_PREFIX}\\bin:${MINGW_PREFIX}\\lib:$PATH fi TAGS="${TAGS},ccalloc" fi diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index 853f03267bd..c801f90d414 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -72,11 +72,18 @@ fi if [[ -n "$DEVTOOLSET_VERSION" ]]; then $PACKAGE_MANAGER install -y centos-release-scl $PACKAGE_MANAGER install -y "devtoolset-$DEVTOOLSET_VERSION" - - # Only add make var if not set - if ! grep -Fq "CXX17=" ~/.R/Makevars &> /dev/null; then + + # Enable devtoolset here so that `which gcc` finds the right compiler below + source /opt/rh/devtoolset-${DEVTOOLSET_VERSION}/enable + + # Build images which require the devtoolset don't have CXX17 variables + # set as the system compiler doesn't support C++17 + if [ ! "`{R_BIN} CMD config CXX17`" ]; then mkdir -p ~/.R - echo "CXX17=g++ -std=gnu++17 -g -O2 -fpic" >> ~/.R/Makevars + echo "CC = $(which gcc) -fPIC" >> ~/.R/Makevars + echo "CXX17 = $(which g++) -fPIC" >> ~/.R/Makevars + echo "CXX17STD = -std=c++17" >> ~/.R/Makevars + echo "CXX17FLAGS = ${CXX11FLAGS}" >> ~/.R/Makevars fi fi diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index f532bc7cf0a..d7df44e2e43 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -26,19 +26,6 @@ pushd ${source_dir} printenv -if [[ -n "$DEVTOOLSET_VERSION" ]]; then - # enable the devtoolset version to use it - source /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable - - # Build images which require the devtoolset don't have CXX17 variables - # set as the system compiler doesn't support C++17 - mkdir -p ~/.R - echo "CC = $(which gcc) -fPIC" >> ~/.R/Makevars - echo "CXX17 = $(which g++) -fPIC" >> ~/.R/Makevars - echo "CXX17STD = -std=c++17" >> ~/.R/Makevars - echo "CXX17FLAGS = ${CXX11FLAGS}" >> ~/.R/Makevars -fi - # Run the nixlibs.R test suite, which is not included in the installed package ${R_BIN} -e 'setwd("tools"); testthat::test_dir(".")' diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index c361af1d267..6b6a5dd0c99 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -23,26 +23,15 @@ set -ex # Make sure it is absolute and exported export ARROW_HOME="$(cd "${ARROW_HOME}" && pwd)" -if [ "$RTOOLS_VERSION" = "35" ]; then - # Use rtools-backports if building with rtools35 - curl https://raw.githubusercontent.com/r-windows/rtools-backports/master/pacman.conf > /etc/pacman.conf - pacman --noconfirm -Syy - # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) - RWINLIB_LIB_DIR="lib-4.9.3" - # This is the default (will build for each arch) but we can set up CI to - # do these in parallel - : ${MINGW_ARCH:="mingw32 mingw64"} -else - # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN - # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf - # curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" - # pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz - # pacman --noconfirm -Scc - - pacman --noconfirm -Syy - RWINLIB_LIB_DIR="lib" - : ${MINGW_ARCH:="mingw32 mingw64 ucrt64"} -fi +# Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN +# curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf +# curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" +# pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz +# pacman --noconfirm -Scc + +pacman --noconfirm -Syy +RWINLIB_LIB_DIR="lib" +: ${MINGW_ARCH:="mingw32 mingw64 ucrt64"} export MINGW_ARCH @@ -78,26 +67,19 @@ fi if [ -d mingw64/lib/ ]; then ls $MSYS_LIB_DIR/mingw64/lib/ # Make the rest of the directory structure - # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) - mkdir -p $DST_DIR/${RWINLIB_LIB_DIR}/x64 - # lib is for the new gcc 8 toolchain (Rtools 4.0) mkdir -p $DST_DIR/lib/x64 # Move the 64-bit versions of libarrow into the expected location - mv mingw64/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/x64 - # These may be from https://dl.bintray.com/rtools/backports/ - cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x64 + mv mingw64/lib/*.a $DST_DIR/lib/x64 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/x64 + cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/x64 fi # Same for the 32-bit versions if [ -d mingw32/lib/ ]; then ls $MSYS_LIB_DIR/mingw32/lib/ - mkdir -p $DST_DIR/${RWINLIB_LIB_DIR}/i386 mkdir -p $DST_DIR/lib/i386 - mv mingw32/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 - cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 - cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/i386 + mv mingw32/lib/*.a $DST_DIR/lib/i386 + cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/i386 fi # Do the same also for ucrt64 diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index cef4eb0b161..1abc6a5fe46 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -400,22 +400,13 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CXX_ONLY_FLAGS "${CXX_ONLY_FLAGS} -Wno-noexcept-type") endif() - if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "5.2") - # Disabling semantic interposition allows faster calling conventions - # when calling global functions internally, and can also help inlining. - # See https://stackoverflow.com/questions/35745543/new-option-in-gcc-5-3-fno-semantic-interposition - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-semantic-interposition") - endif() - - if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "4.9") - # Add colors when paired with ninja - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") - endif() + # Disabling semantic interposition allows faster calling conventions + # when calling global functions internally, and can also help inlining. + # See https://stackoverflow.com/questions/35745543/new-option-in-gcc-5-3-fno-semantic-interposition + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-semantic-interposition") - if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "6.0") - # Work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=43407 - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-attributes") - endif() + # Add colors when paired with ninja + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") if(CMAKE_UNITY_BUILD) # Work around issue similar to https://bugs.webkit.org/show_bug.cgi?id=176869 @@ -507,18 +498,12 @@ if(ARROW_CPU_FLAG STREQUAL "armv8") add_definitions(-DARROW_HAVE_NEON) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS - "5.4") - message(WARNING "Disable Armv8 CRC and Crypto as compiler doesn't support them well." - ) - else() - if(ARROW_ARMV8_ARCH_FLAG MATCHES "\\+crypto") - add_definitions(-DARROW_HAVE_ARMV8_CRYPTO) - endif() - # armv8.1+ implies crc support - if(ARROW_ARMV8_ARCH_FLAG MATCHES "armv8\\.[1-9]|\\+crc") - add_definitions(-DARROW_HAVE_ARMV8_CRC) - endif() + if(ARROW_ARMV8_ARCH_FLAG MATCHES "\\+crypto") + add_definitions(-DARROW_HAVE_ARMV8_CRYPTO) + endif() + # armv8.1+ implies crc support + if(ARROW_ARMV8_ARCH_FLAG MATCHES "armv8\\.[1-9]|\\+crc") + add_definitions(-DARROW_HAVE_ARMV8_CRC) endif() elseif(NOT ARROW_SIMD_LEVEL STREQUAL "NONE") message(WARNING "ARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL} not supported by Arm.") diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 9b6cc4865f3..65925dd7b0f 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -652,18 +652,9 @@ endif() if(DEFINED ENV{ARROW_SNAPPY_URL}) set(SNAPPY_SOURCE_URL "$ENV{ARROW_SNAPPY_URL}") else() - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS - "4.9") - # There is a bug in GCC < 4.9 with Snappy 1.1.9, so revert to 1.1.8 "SNAPPY_OLD" for those (ARROW-14661) - set_urls(SNAPPY_SOURCE_URL - "https://github.com/google/snappy/archive/${ARROW_SNAPPY_OLD_BUILD_VERSION}.tar.gz" - "${THIRDPARTY_MIRROR_URL}/snappy-${ARROW_SNAPPY_OLD_BUILD_VERSION}.tar.gz") - set(ARROW_SNAPPY_BUILD_SHA256_CHECKSUM ${ARROW_SNAPPY_OLD_BUILD_SHA256_CHECKSUM}) - else() - set_urls(SNAPPY_SOURCE_URL - "https://github.com/google/snappy/archive/${ARROW_SNAPPY_BUILD_VERSION}.tar.gz" - "${THIRDPARTY_MIRROR_URL}/snappy-${ARROW_SNAPPY_BUILD_VERSION}.tar.gz") - endif() + set_urls(SNAPPY_SOURCE_URL + "https://github.com/google/snappy/archive/${ARROW_SNAPPY_BUILD_VERSION}.tar.gz" + "${THIRDPARTY_MIRROR_URL}/snappy-${ARROW_SNAPPY_BUILD_VERSION}.tar.gz") endif() # Remove these two lines once https://github.com/substrait-io/substrait/pull/342 merges @@ -3943,7 +3934,7 @@ macro(build_grpc) gRPC::grpc gRPC::grpcpp_for_bundling gRPC::upb) - if(ABS_VENDORED) + if(ABSL_VENDORED) list(APPEND ARROW_BUNDLED_STATIC_LIBS ${GRPC_GPR_ABSL_LIBRARIES}) endif() endmacro() @@ -4618,10 +4609,6 @@ endif() macro(build_awssdk) message(STATUS "Building AWS C++ SDK from source") - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS - "4.9") - message(FATAL_ERROR "AWS C++ SDK requires gcc >= 4.9") - endif() set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include") set(AWSSDK_LIB_DIR "lib") diff --git a/cpp/proto/substrait/extension_rels.proto b/cpp/proto/substrait/extension_rels.proto index 518412969f5..6f806d00e5b 100644 --- a/cpp/proto/substrait/extension_rels.proto +++ b/cpp/proto/substrait/extension_rels.proto @@ -16,7 +16,7 @@ // under the License. syntax = "proto3"; -package arrow.substrait_ext; +package arrow.substrait; import "substrait/algebra.proto"; @@ -25,12 +25,20 @@ option go_package = "github.com/apache/arrow/substrait"; option java_multiple_files = true; option java_package = "io.arrow.substrait"; +// As-Of-Join relation message AsOfJoinRel { - repeated AsOfJoinKeys input_keys = 1; + // One key per input relation, each key describing how to join the corresponding input + repeated AsOfJoinKey keys = 1; + + // As-Of tolerance, in units of the on-key int64 tolerance = 2; - message AsOfJoinKeys { + // As-Of-Join key + message AsOfJoinKey { + // A field reference defining the on-key .substrait.Expression on = 1; + + // A set of field references defining the by-key repeated .substrait.Expression by = 2; } } diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc index 5af5ebccc84..18f88bc6dfb 100644 --- a/cpp/src/arrow/adapters/orc/adapter.cc +++ b/cpp/src/arrow/adapters/orc/adapter.cc @@ -411,7 +411,7 @@ class ORCFileReader::Impl { ARROW_RETURN_IF(*it < 0, Status::Invalid("Negative field index")); include_indices_list.push_back(*it); } - opts->includeTypes(include_indices_list); + opts->include(include_indices_list); return Status::OK(); } diff --git a/cpp/src/arrow/adapters/orc/adapter_test.cc b/cpp/src/arrow/adapters/orc/adapter_test.cc index 5c234cc97c4..afc4bdb1d3b 100644 --- a/cpp/src/arrow/adapters/orc/adapter_test.cc +++ b/cpp/src/arrow/adapters/orc/adapter_test.cc @@ -226,7 +226,8 @@ std::shared_ptr GenerateRandomTable(const std::shared_ptr& schema void AssertTableWriteReadEqual(const std::shared_ptr
& input_table, const std::shared_ptr
& expected_output_table, - const int64_t max_size = kDefaultSmallMemStreamSize) { + const int64_t max_size = kDefaultSmallMemStreamSize, + std::vector* opt_selected_read_indices = nullptr) { EXPECT_OK_AND_ASSIGN(auto buffer_output_stream, io::BufferOutputStream::Create(max_size)); auto write_options = adapters::orc::WriteOptions(); @@ -250,7 +251,11 @@ void AssertTableWriteReadEqual(const std::shared_ptr
& input_table, ASSERT_EQ(reader->GetCompression(), write_options.compression); ASSERT_EQ(reader->GetCompressionSize(), write_options.compression_block_size); ASSERT_EQ(reader->GetRowIndexStride(), write_options.row_index_stride); - EXPECT_OK_AND_ASSIGN(auto actual_output_table, reader->Read()); + EXPECT_OK_AND_ASSIGN(auto actual_output_table, + opt_selected_read_indices == nullptr + ? reader->Read() + : reader->Read(*opt_selected_read_indices)); + ASSERT_OK(actual_output_table->ValidateFull()); AssertTablesEqual(*expected_output_table, *actual_output_table, false, false); } @@ -451,6 +456,37 @@ TEST_F(TestORCWriterTrivialNoConversion, writeChunkless) { std::shared_ptr
table = TableFromJSON(table_schema, {}); AssertTableWriteReadEqual(table, table, kDefaultSmallMemStreamSize / 16); } +TEST_F(TestORCWriterTrivialNoConversion, writeTrivialChunkAndSelectField) { + std::shared_ptr
table = TableFromJSON(table_schema, {R"([])"}); + std::shared_ptr schema_selected = + schema({field("int8", int8()), field("int32", int32())}); + std::shared_ptr
table_selected = TableFromJSON(schema_selected, {R"([])"}); + std::vector selected_indices = {1, 3}; + AssertTableWriteReadEqual(table, table_selected, kDefaultSmallMemStreamSize / 16, + &selected_indices); +} +TEST_F(TestORCWriterTrivialNoConversion, writeFilledChunkAndSelectField) { + std::vector selected_indices = {1, 7}; + random::RandomArrayGenerator rand(kRandomSeed); + std::shared_ptr local_schema = schema({ + field("bool", boolean()), + field("int32", int32()), + field("int64", int64()), + field("float", float32()), + field("struct", struct_({field("a", utf8()), field("b", int64())})), + field("double", float64()), + field("date32", date32()), + field("ts3", timestamp(TimeUnit::NANO)), + field("string", utf8()), + field("binary", binary()), + }); + auto batch = rand.BatchOf(local_schema->fields(), 100); + std::shared_ptr
table = Table::Make(local_schema, batch->columns()); + EXPECT_OK_AND_ASSIGN(auto table_selected, table->SelectColumns(selected_indices)); + AssertTableWriteReadEqual(table, table_selected, kDefaultSmallMemStreamSize, + &selected_indices); +} + class TestORCWriterTrivialWithConversion : public ::testing::Test { public: TestORCWriterTrivialWithConversion() { diff --git a/cpp/src/arrow/compute/exec/asof_join_benchmark.cc b/cpp/src/arrow/compute/exec/asof_join_benchmark.cc index 3c6b78d29f1..1e485401d6b 100644 --- a/cpp/src/arrow/compute/exec/asof_join_benchmark.cc +++ b/cpp/src/arrow/compute/exec/asof_join_benchmark.cc @@ -115,6 +115,15 @@ AsofJoinNodeOptions GetRepeatedOptions(size_t repeat, FieldRef on_key, return AsofJoinNodeOptions(input_keys, tolerance); } +AsofJoinNodeOptions GetRepeatedOptions(size_t repeat, FieldRef on_key, + std::vector by_key, int64_t tolerance) { + std::vector input_keys(repeat); + for (size_t i = 0; i < repeat; i++) { + input_keys[i] = {on_key, by_key}; + } + return AsofJoinNodeOptions(input_keys, tolerance); +} + static void AsOfJoinOverhead(benchmark::State& state) { int64_t tolerance = 0; AsofJoinNodeOptions options = diff --git a/cpp/src/arrow/dataset/file_parquet.cc b/cpp/src/arrow/dataset/file_parquet.cc index f07254e1115..da9748de96e 100644 --- a/cpp/src/arrow/dataset/file_parquet.cc +++ b/cpp/src/arrow/dataset/file_parquet.cc @@ -742,16 +742,7 @@ Result> ParquetFileFragment::TryCountRows( compute::Expression predicate) { DCHECK_NE(metadata_, nullptr); if (ExpressionHasFieldRefs(predicate)) { -#if defined(__GNUC__) && (__GNUC__ < 5) - // ARROW-12694: with GCC 4.9 (RTools 35) we sometimes segfault here if we move(result) - auto result = TestRowGroups(std::move(predicate)); - if (!result.ok()) { - return result.status(); - } - auto expressions = result.ValueUnsafe(); -#else ARROW_ASSIGN_OR_RAISE(auto expressions, TestRowGroups(std::move(predicate))); -#endif int64_t rows = 0; for (size_t i = 0; i < row_groups_->size(); i++) { // If the row group is entirely excluded, exclude it from the row count diff --git a/cpp/src/arrow/dataset/scanner.cc b/cpp/src/arrow/dataset/scanner.cc index 08c9b833b1c..88d28a17ed6 100644 --- a/cpp/src/arrow/dataset/scanner.cc +++ b/cpp/src/arrow/dataset/scanner.cc @@ -786,7 +786,19 @@ Result ProjectionDescr::FromNames(std::vector name const Schema& dataset_schema) { std::vector exprs(names.size()); for (size_t i = 0; i < exprs.size(); ++i) { - exprs[i] = compute::field_ref(names[i]); + // If name isn't in schema, try finding it by dotted path. + if (dataset_schema.GetFieldByName(names[i]) == nullptr) { + auto name = names[i]; + if (name.rfind(".", 0) != 0) { + name = "." + name; + } + ARROW_ASSIGN_OR_RAISE(auto field_ref, FieldRef::FromDotPath(name)); + // safe as we know there is at least 1 dot. + names[i] = name.substr(name.rfind(".") + 1); + exprs[i] = compute::field_ref(field_ref); + } else { + exprs[i] = compute::field_ref(names[i]); + } } auto fields = dataset_schema.fields(); for (const auto& aug_field : kAugmentedFields) { diff --git a/cpp/src/arrow/dataset/scanner_test.cc b/cpp/src/arrow/dataset/scanner_test.cc index 59fb8089c0a..73694f4b33a 100644 --- a/cpp/src/arrow/dataset/scanner_test.cc +++ b/cpp/src/arrow/dataset/scanner_test.cc @@ -1088,6 +1088,24 @@ TEST_P(TestScanner, ProjectedScanNested) { AssertScanBatchesUnorderedEqualRepetitionsOf(MakeScanner(batch_in), batch_out); } +TEST_P(TestScanner, ProjectedScanNestedFromNames) { + SetSchema({ + field("struct", struct_({field("i32", int32()), field("f64", float64())})), + field("nested", struct_({field("left", int32()), + field("right", struct_({field("i32", int32()), + field("f64", float64())}))})), + }); + ASSERT_OK_AND_ASSIGN(auto descr, + ProjectionDescr::FromNames({".struct.i32", "nested.right.f64"}, + *options_->dataset_schema)) + SetProjection(options_.get(), std::move(descr)); + auto batch_in = ConstantArrayGenerator::Zeroes(GetParam().items_per_batch, schema_); + auto batch_out = ConstantArrayGenerator::Zeroes( + GetParam().items_per_batch, + schema({field("i32", int32()), field("f64", float64())})); + AssertScanBatchesUnorderedEqualRepetitionsOf(MakeScanner(batch_in), batch_out); +} + TEST_P(TestScanner, MaterializeMissingColumn) { SetSchema({field("i32", int32()), field("f64", float64())}); auto batch_missing_f64 = ConstantArrayGenerator::Zeroes( diff --git a/cpp/src/arrow/dataset/test_util.h b/cpp/src/arrow/dataset/test_util.h index ee9c2fcad8f..8764eed27aa 100644 --- a/cpp/src/arrow/dataset/test_util.h +++ b/cpp/src/arrow/dataset/test_util.h @@ -157,7 +157,7 @@ class DatasetFixtureMixin : public ::testing::Test { std::shared_ptr lhs; ASSERT_OK(expected->ReadNext(&lhs)); EXPECT_NE(lhs, nullptr); - AssertBatchesEqual(*lhs, batch); + AssertBatchesEqual(*lhs, batch, true); } /// \brief Ensure that record batches found in reader are equals to the diff --git a/cpp/src/arrow/engine/substrait/expression_internal.cc b/cpp/src/arrow/engine/substrait/expression_internal.cc index 343d8cd9ee6..823576003c4 100644 --- a/cpp/src/arrow/engine/substrait/expression_internal.cc +++ b/cpp/src/arrow/engine/substrait/expression_internal.cc @@ -37,6 +37,8 @@ using internal::checked_cast; namespace engine { +namespace substrait = ::substrait; + namespace { Id NormalizeFunctionName(Id id) { diff --git a/cpp/src/arrow/engine/substrait/expression_internal.h b/cpp/src/arrow/engine/substrait/expression_internal.h index f132afc0c1a..6eb7d4117c8 100644 --- a/cpp/src/arrow/engine/substrait/expression_internal.h +++ b/cpp/src/arrow/engine/substrait/expression_internal.h @@ -32,6 +32,8 @@ namespace arrow { namespace engine { +namespace substrait = ::substrait; + ARROW_ENGINE_EXPORT Result FromProto(const substrait::Expression&, const ExtensionSet&, const ConversionOptions&); diff --git a/cpp/src/arrow/engine/substrait/options.cc b/cpp/src/arrow/engine/substrait/options.cc index 6614814771b..f7c8bf4713e 100644 --- a/cpp/src/arrow/engine/substrait/options.cc +++ b/cpp/src/arrow/engine/substrait/options.cc @@ -28,13 +28,15 @@ namespace arrow { namespace engine { +namespace substrait = ::substrait; + class DefaultExtensionProvider : public ExtensionProvider { public: Result MakeRel(const std::vector& inputs, const google::protobuf::Any& rel, const ExtensionSet& ext_set) override { - if (rel.Is()) { - substrait_ext::AsOfJoinRel as_of_join_rel; + if (rel.Is()) { + arrow::substrait::AsOfJoinRel as_of_join_rel; rel.UnpackTo(&as_of_join_rel); return MakeAsOfJoinRel(inputs, as_of_join_rel, ext_set); } @@ -45,18 +47,18 @@ class DefaultExtensionProvider : public ExtensionProvider { private: Result MakeAsOfJoinRel( const std::vector& inputs, - const substrait_ext::AsOfJoinRel& as_of_join_rel, const ExtensionSet& ext_set) { + const arrow::substrait::AsOfJoinRel& as_of_join_rel, const ExtensionSet& ext_set) { if (inputs.size() < 2) { return Status::Invalid("substrait::AsOfJoinNode too few input tables: ", inputs.size()); } - if (static_cast(as_of_join_rel.input_keys_size()) != inputs.size()) { + if (static_cast(as_of_join_rel.keys_size()) != inputs.size()) { return Status::Invalid("substrait::AsOfJoinNode mismatched number of inputs"); } size_t n_input = inputs.size(), i = 0; std::vector input_keys(n_input); - for (const auto& keys : as_of_join_rel.input_keys()) { + for (const auto& keys : as_of_join_rel.keys()) { // on-key if (!keys.has_on()) { return Status::Invalid("substrait::AsOfJoinNode missing on-key for input ", i); diff --git a/cpp/src/arrow/engine/substrait/options.h b/cpp/src/arrow/engine/substrait/options.h index 2f46abbbfb8..57f29f65630 100644 --- a/cpp/src/arrow/engine/substrait/options.h +++ b/cpp/src/arrow/engine/substrait/options.h @@ -36,7 +36,7 @@ namespace engine { /// How strictly to adhere to the input structure when converting between Substrait and /// Acero representations of a plan. This allows the user to trade conversion accuracy /// for performance and lenience. -enum class ConversionStrictness { +enum class ARROW_ENGINE_EXPORT ConversionStrictness { /// When a primitive is used at the input that doesn't have an exact match at the /// output, reject the conversion. This effectively asserts that there is no (known) /// information loss in the conversion, and that plans should either round-trip back and diff --git a/cpp/src/arrow/engine/substrait/plan_internal.cc b/cpp/src/arrow/engine/substrait/plan_internal.cc index 64f129101f4..4b1abb7677b 100644 --- a/cpp/src/arrow/engine/substrait/plan_internal.cc +++ b/cpp/src/arrow/engine/substrait/plan_internal.cc @@ -34,6 +34,8 @@ using internal::checked_cast; namespace engine { +namespace substrait = ::substrait; + Status AddExtensionSetToPlan(const ExtensionSet& ext_set, substrait::Plan* plan) { plan->clear_extension_uris(); diff --git a/cpp/src/arrow/engine/substrait/plan_internal.h b/cpp/src/arrow/engine/substrait/plan_internal.h index 7d6dd375288..473edac7239 100644 --- a/cpp/src/arrow/engine/substrait/plan_internal.h +++ b/cpp/src/arrow/engine/substrait/plan_internal.h @@ -30,6 +30,8 @@ namespace arrow { namespace engine { +namespace substrait = ::substrait; + /// \brief Replaces the extension information of a Substrait Plan message with the given /// extension set, such that the anchors defined therein can be used in the rest of the /// plan. diff --git a/cpp/src/arrow/engine/substrait/relation_internal.cc b/cpp/src/arrow/engine/substrait/relation_internal.cc index d06cc7ef6f2..0363d382f10 100644 --- a/cpp/src/arrow/engine/substrait/relation_internal.cc +++ b/cpp/src/arrow/engine/substrait/relation_internal.cc @@ -43,6 +43,8 @@ using internal::UriFromAbsolutePath; namespace engine { +namespace substrait = ::substrait; + struct EmitInfo { std::vector expressions; std::shared_ptr schema; diff --git a/cpp/src/arrow/engine/substrait/relation_internal.h b/cpp/src/arrow/engine/substrait/relation_internal.h index c7241490783..3f48cb55e62 100644 --- a/cpp/src/arrow/engine/substrait/relation_internal.h +++ b/cpp/src/arrow/engine/substrait/relation_internal.h @@ -31,6 +31,8 @@ namespace arrow { namespace engine { +namespace substrait = ::substrait; + /// Information resulting from converting a Substrait relation. struct ARROW_ENGINE_EXPORT DeclarationInfo { /// The compute declaration produced thus far. diff --git a/cpp/src/arrow/engine/substrait/serde.cc b/cpp/src/arrow/engine/substrait/serde.cc index b2cac3f82e4..22ef0d38f7a 100644 --- a/cpp/src/arrow/engine/substrait/serde.cc +++ b/cpp/src/arrow/engine/substrait/serde.cc @@ -42,6 +42,8 @@ namespace arrow { namespace engine { +namespace substrait = ::substrait; + Status ParseFromBufferImpl(const Buffer& buf, const std::string& full_name, google::protobuf::Message* message) { google::protobuf::io::ArrayInputStream buf_stream{buf.data(), diff --git a/cpp/src/arrow/engine/substrait/serde_test.cc b/cpp/src/arrow/engine/substrait/serde_test.cc index 09d260aadf6..fb4454c8d97 100644 --- a/cpp/src/arrow/engine/substrait/serde_test.cc +++ b/cpp/src/arrow/engine/substrait/serde_test.cc @@ -2255,16 +2255,6 @@ TEST(SubstraitRoundTrip, BasicPlanEndToEnd) { EXPECT_TRUE(expected_table->Equals(*rnd_trp_table)); } -NamedTableProvider ProvideMadeTable( - std::function>(const std::vector&)> make) { - return [make](const std::vector& names) -> Result { - ARROW_ASSIGN_OR_RAISE(auto table, make(names)); - std::shared_ptr options = - std::make_shared(table); - return compute::Declaration("table_source", {}, options, "mock_source"); - }; -} - TEST(SubstraitRoundTrip, ProjectRel) { #ifdef _WIN32 GTEST_SKIP() << "ARROW-16392: Substrait File URI not supported for Windows"; @@ -3330,6 +3320,16 @@ TEST(Substrait, IsthmusPlan) { *compute::default_exec_context(), buf, {}, conversion_options); } +NamedTableProvider ProvideMadeTable( + std::function>(const std::vector&)> make) { + return [make](const std::vector& names) -> Result { + ARROW_ASSIGN_OR_RAISE(auto table, make(names)); + std::shared_ptr options = + std::make_shared(table); + return compute::Declaration("table_source", {}, options, "mock_source"); + }; +} + TEST(Substrait, ProjectWithMultiFieldExpressions) { compute::ExecContext exec_context; auto dummy_schema = @@ -3659,7 +3659,7 @@ TEST(Substrait, NestedEmitProjectWithMultiFieldExpressions) { buf, {}, conversion_options); } -TEST(Substrait, PlanWithExtension) { +TEST(Substrait, PlanWithAsOfJoinExtension) { // This demos an extension relation std::string substrait_json = R"({ "extensionUris": [], @@ -3752,8 +3752,8 @@ TEST(Substrait, PlanWithExtension) { } ], "detail": { - "@type": "/arrow.substrait_ext.AsOfJoinRel", - "input_keys" : [ + "@type": "/arrow.substrait.AsOfJoinRel", + "keys" : [ { "on": { "selection": { diff --git a/cpp/src/arrow/engine/substrait/test_plan_builder.cc b/cpp/src/arrow/engine/substrait/test_plan_builder.cc index 79820672ed9..a6b2d7c6780 100644 --- a/cpp/src/arrow/engine/substrait/test_plan_builder.cc +++ b/cpp/src/arrow/engine/substrait/test_plan_builder.cc @@ -31,6 +31,8 @@ namespace arrow { namespace engine { + +namespace substrait = ::substrait; namespace internal { static const ConversionOptions kPlanBuilderConversionOptions; diff --git a/cpp/src/arrow/engine/substrait/type_internal.h b/cpp/src/arrow/engine/substrait/type_internal.h index 6db9aea01ae..9b4132c74c2 100644 --- a/cpp/src/arrow/engine/substrait/type_internal.h +++ b/cpp/src/arrow/engine/substrait/type_internal.h @@ -31,6 +31,8 @@ namespace arrow { namespace engine { +namespace substrait = ::substrait; + ARROW_ENGINE_EXPORT Result, bool>> FromProto(const substrait::Type&, const ExtensionSet&, diff --git a/cpp/src/parquet/column_reader_test.cc b/cpp/src/parquet/column_reader_test.cc index e7162eb981c..b2f947eea46 100644 --- a/cpp/src/parquet/column_reader_test.cc +++ b/cpp/src/parquet/column_reader_test.cc @@ -260,7 +260,8 @@ TEST_F(TestPrimitiveReader, TestInt32FlatRepeated) { ASSERT_NO_FATAL_FAILURE(ExecuteDict(num_pages, levels_per_page, &descr)); } -TEST_F(TestPrimitiveReader, TestInt32FlatRequiredSkip) { +// Tests skipping around page boundaries. +TEST_F(TestPrimitiveReader, TestSkipAroundPageBoundries) { int levels_per_page = 100; int num_pages = 5; max_def_level_ = 0; @@ -289,10 +290,10 @@ TEST_F(TestPrimitiveReader, TestInt32FlatRequiredSkip) { values_.begin() + static_cast(2.5 * static_cast(levels_per_page))); ASSERT_TRUE(vector_equal(sub_values, vresult)); - // 2) skip_size == page_size (skip across two pages) + // 2) skip_size == page_size (skip across two pages from page 2.5 to 3.5) levels_skipped = reader->Skip(levels_per_page); ASSERT_EQ(levels_per_page, levels_skipped); - // Read half a page + // Read half a page (page 3.5 to 4) reader->ReadBatch(levels_per_page / 2, dresult.data(), rresult.data(), vresult.data(), &values_read); sub_values.clear(); @@ -303,10 +304,10 @@ TEST_F(TestPrimitiveReader, TestInt32FlatRequiredSkip) { ASSERT_TRUE(vector_equal(sub_values, vresult)); // 3) skip_size < page_size (skip limited to a single page) - // Skip half a page + // Skip half a page (page 4 to 4.5) levels_skipped = reader->Skip(levels_per_page / 2); ASSERT_EQ(0.5 * levels_per_page, levels_skipped); - // Read half a page + // Read half a page (page 4.5 to 5) reader->ReadBatch(levels_per_page / 2, dresult.data(), rresult.data(), vresult.data(), &values_read); sub_values.clear(); @@ -316,6 +317,15 @@ TEST_F(TestPrimitiveReader, TestInt32FlatRequiredSkip) { values_.end()); ASSERT_TRUE(vector_equal(sub_values, vresult)); + // 4) skip_size = 0 + levels_skipped = reader->Skip(0); + ASSERT_EQ(0, levels_skipped); + + // 5) Skip past the end page. There are 5 pages and we have either skipped + // or read all of them, so there is nothing left to skip. + levels_skipped = reader->Skip(10); + ASSERT_EQ(0, levels_skipped); + values_.clear(); def_levels_.clear(); rep_levels_.clear(); @@ -323,6 +333,55 @@ TEST_F(TestPrimitiveReader, TestInt32FlatRequiredSkip) { reader_.reset(); } +// Skip with repeated field. This test makes it clear that we are skipping +// values and not records. +TEST_F(TestPrimitiveReader, TestSkipRepeatedField) { + // Example schema: message M { repeated int32 b = 1 } + max_def_level_ = 1; + max_rep_level_ = 1; + NodePtr type = schema::Int32("b", Repetition::REPEATED); + const ColumnDescriptor descr(type, max_def_level_, max_rep_level_); + // Example rows: {}, {[10, 10]}, {[20, 20, 20]} + std::vector values = {10, 10, 20, 20, 20}; + std::vector def_levels = {0, 1, 1, 1, 1, 1}; + std::vector rep_levels = {0, 0, 1, 0, 1, 1}; + num_values_ = static_cast(def_levels.size()); + std::shared_ptr page = MakeDataPage( + &descr, values, num_values_, Encoding::PLAIN, /*indices=*/{}, + /*indices_size=*/0, def_levels, max_def_level_, rep_levels, max_rep_level_); + + pages_.push_back(std::move(page)); + + InitReader(&descr); + Int32Reader* reader = static_cast(reader_.get()); + + // Vecotrs to hold read values, definition levels, and repetition levels. + std::vector read_vals(4, -1); + std::vector read_defs(4, -1); + std::vector read_reps(4, -1); + + // Skip two levels. + int64_t levels_skipped = reader->Skip(2); + ASSERT_EQ(2, levels_skipped); + + int64_t num_read_values = 0; + // Read the next set of values + reader->ReadBatch(10, read_defs.data(), read_reps.data(), read_vals.data(), + &num_read_values); + ASSERT_EQ(num_read_values, 4); + // Note that we end up in the record with {[10, 10]} + ASSERT_TRUE(vector_equal({10, 20, 20, 20}, read_vals)); + ASSERT_TRUE(vector_equal({1, 1, 1, 1}, read_defs)); + ASSERT_TRUE(vector_equal({1, 0, 1, 1}, read_reps)); + + // No values remain in data page + levels_skipped = reader->Skip(2); + ASSERT_EQ(0, levels_skipped); + reader->ReadBatch(10, read_defs.data(), read_reps.data(), read_vals.data(), + &num_read_values); + ASSERT_EQ(num_read_values, 0); +} + // Page claims to have two values but only 1 is present. TEST_F(TestPrimitiveReader, TestReadValuesMissing) { max_def_level_ = 1; diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 7409b66c86e..0cc496e9385 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -81,9 +81,6 @@ ARROW_RE2_BUILD_SHA256_CHECKSUM=f89c61410a072e5cbcf8c27e3a778da7d6fd2f2b5b1445cd # 1.1.9 is patched to implement https://github.com/google/snappy/pull/148 if this is bumped, remove the patch ARROW_SNAPPY_BUILD_VERSION=1.1.9 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7 -# There is a bug in GCC < 4.9 with Snappy 1.1.9, so revert to 1.1.8 for those (ARROW-14661) -ARROW_SNAPPY_OLD_BUILD_VERSION=1.1.8 -ARROW_SNAPPY_OLD_BUILD_SHA256_CHECKSUM=16b677f07832a612b0836178db7f374e414f94657c138e6993cbfc5dcc58651f ARROW_SUBSTRAIT_BUILD_VERSION=v0.6.0 ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=7b8583b9684477e9027f417bbfb4febb8acfeb01923dcaa7cf0fd3f921d69c88 ARROW_THRIFT_BUILD_VERSION=0.16.0 @@ -94,8 +91,8 @@ ARROW_UTF8PROC_BUILD_VERSION=v2.7.0 ARROW_UTF8PROC_BUILD_SHA256_CHECKSUM=4bb121e297293c0fd55f08f83afab6d35d48f0af4ecc07523ad8ec99aa2b12a1 ARROW_XSIMD_BUILD_VERSION=9.0.1 ARROW_XSIMD_BUILD_SHA256_CHECKSUM=b1bb5f92167fd3a4f25749db0be7e61ed37e0a5d943490f3accdcd2cd2918cc0 -ARROW_ZLIB_BUILD_VERSION=1.2.12 -ARROW_ZLIB_BUILD_SHA256_CHECKSUM=91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9 +ARROW_ZLIB_BUILD_VERSION=1.2.13 +ARROW_ZLIB_BUILD_SHA256_CHECKSUM=b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30 ARROW_ZSTD_BUILD_VERSION=v1.5.2 ARROW_ZSTD_BUILD_SHA256_CHECKSUM=f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index c8ea8a13a4b..a83c190d121 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -738,14 +738,16 @@ class Target(Serializable): (currently only an email address where the notification should be sent). """ - def __init__(self, head, branch, remote, version, email=None): + def __init__(self, head, branch, remote, version, r_version, email=None): self.head = head self.email = email self.branch = branch self.remote = remote self.github_repo = "/".join(_parse_github_user_repo(remote)) self.version = version + self.r_version = r_version self.no_rc_version = re.sub(r'-rc\d+\Z', '', version) + self.no_rc_r_version = re.sub(r'-rc\d+\Z', '', r_version) # TODO(ARROW-17552): Remove "master" from default_branch after # migration to "main". self.default_branch = ['main', 'master'] @@ -791,8 +793,39 @@ def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, if email is None: email = repo.user_email + version_dev_match = re.match(r".*\.dev(\d+)$", version) + if version_dev_match: + with open(f"{repo.path}/r/DESCRIPTION") as description_file: + description = description_file.read() + r_version_pattern = re.compile(r"^Version:\s*(.*)$", + re.MULTILINE) + r_version = re.findall(r_version_pattern, description)[0] + if r_version: + version_dev = int(version_dev_match[1]) + # "1_0000_00_00 +" is for generating a greater version + # than YYYYMMDD. For example, 1_0000_00_01 + # (version_dev == 1 case) is greater than 2022_10_16. + # + # Why do we need a greater version than YYYYMMDD? It's + # for keeping backward compatibility. We used + # MAJOR.MINOR.PATCH.YYYYMMDD as our nightly package + # version. (See also ARROW-16403). If we use "9000 + + # version_dev" here, a developer that used + # 9.0.0.20221016 can't upgrade to the later nightly + # package unless we release 10.0.0. Because 9.0.0.9234 + # or something is less than 9.0.0.20221016. + r_version_dev = 1_0000_00_00 + version_dev + # version: 10.0.0.dev234 + # r_version: 9.0.0.9000 + # -> 9.0.0.100000234 + r_version = re.sub(r"\.9000\Z", f".{r_version_dev}", r_version) + else: + r_version = version + else: + r_version = version + return cls(head=head, email=email, branch=branch, remote=remote, - version=version) + version=version, r_version=r_version) def is_default_branch(self): # TODO(ARROW-17552): Switch the condition to "is" instead of "in" @@ -1105,7 +1138,10 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None): 'version': target.version, 'no_rc_version': target.no_rc_version, 'no_rc_semver_version': target.no_rc_semver_version, - 'no_rc_snapshot_version': target.no_rc_snapshot_version} + 'no_rc_snapshot_version': target.no_rc_snapshot_version, + 'r_version': target.r_version, + 'no_rc_r_version': target.no_rc_r_version, + } for task_name, task in task_definitions.items(): task = task.copy() artifacts = task.pop('artifacts', None) or [] # because of yaml @@ -1260,6 +1296,7 @@ def validate(self): branch='master', remote='https://github.com/apache/arrow', version='1.0.0dev123', + r_version='0.13.0.100000123', email='dummy@example.ltd' ) job = Job.from_config(config=self, diff --git a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml index 15e8ca3ff5e..eb03bbee0bd 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml +++ b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job-no-failure.yaml @@ -5,7 +5,9 @@ target: !Target branch: refs/pull/4435/merge remote: https://github.com/apache/arrow version: 0.13.0.dev306 + r_version: 0.12.0.100000306 no_rc_version: 0.13.0.dev306 + no_rc_r_version: 0.12.0.100000306 tasks: docker-cpp-cmake32: !Task ci: circle @@ -64,4 +66,4 @@ branch: ursabot-1 _queue: !Queue path: the_path github_token: xxxxxxxxx - _remote_url: https://github.com/apache/crossbow \ No newline at end of file + _remote_url: https://github.com/apache/crossbow diff --git a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml index 90eab704988..f6de07dd456 100644 --- a/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml +++ b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml @@ -5,7 +5,9 @@ target: !Target branch: refs/pull/4435/merge remote: https://github.com/apache/arrow version: 0.13.0.dev306 + r_version: 0.12.0.100000306 no_rc_version: 0.13.0.dev306 + no_rc_r_version: 0.12.0.100000306 tasks: docker-cpp-cmake32: !Task ci: circle @@ -64,4 +66,4 @@ branch: ursabot-1 _queue: !Queue path: the_path github_token: xxxxxxxxx - _remote_url: https://github.com/apache/crossbow \ No newline at end of file + _remote_url: https://github.com/apache/crossbow diff --git a/dev/archery/archery/docker/cli.py b/dev/archery/archery/docker/cli.py index bbdd2261db6..c7b42c094f6 100644 --- a/dev/archery/archery/docker/cli.py +++ b/dev/archery/archery/docker/cli.py @@ -16,6 +16,7 @@ # under the License. import os +import sys import click @@ -289,3 +290,27 @@ def docker_compose_images(obj): click.echo('Available images:') for image in compose.images(): click.echo(f' - {image}') + + +@docker.command('info') +@click.argument('service_name') +@click.option('--show', '-s', required=False, + help="Show only specific docker-compose key. Examples of keys:" + " command, environment, build, dockerfile") +@click.pass_obj +def docker_compose_info(obj, service_name, show): + """Show docker-compose definition info for service_name. + + SERVICE_NAME is the name of the docker service defined on + the docker-compose. Look at `archery docker images` output for names. + """ + compose = obj['compose'] + try: + service = compose.config.raw_config["services"][service_name] + except KeyError: + click.echo(f'Service name {service_name} could not be found', err=True) + sys.exit(1) + else: + click.echo(f'Service {service_name} docker-compose config:') + output = "\n".join(compose.info(service, show)) + click.echo(output) diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py index da15f86935b..de5e6cf41c9 100644 --- a/dev/archery/archery/docker/core.py +++ b/dev/archery/archery/docker/core.py @@ -95,12 +95,12 @@ def _read_config(self, config_path, compose_bin): """ yaml = YAML() with config_path.open() as fp: - config = yaml.load(fp) + self.raw_config = yaml.load(fp) - services = config['services'].keys() - self.hierarchy = dict(flatten(config.get('x-hierarchy', {}))) - self.limit_presets = config.get('x-limit-presets', {}) - self.with_gpus = config.get('x-with-gpus', []) + services = self.raw_config['services'].keys() + self.hierarchy = dict(flatten(self.raw_config.get('x-hierarchy', {}))) + self.limit_presets = self.raw_config.get('x-limit-presets', {}) + self.with_gpus = self.raw_config.get('x-with-gpus', []) nodes = self.hierarchy.keys() errors = [] @@ -417,3 +417,22 @@ def _push(service): def images(self): return sorted(self.config.hierarchy.keys()) + + def info(self, key_name, filters=None, prefix=' '): + output = [] + for key, value in key_name.items(): + if hasattr(value, 'items'): + temp_filters = filters + if key == filters or filters is None: + output.append(f'{prefix} {key}') + # Keep showing this specific key + # as parent matched filter + temp_filters = None + output.extend(self.info(value, temp_filters, prefix + " ")) + else: + if key == filters or filters is None: + output.append( + f'{prefix} {key}: ' + + f'{value if value is not None else ""}' + ) + return output diff --git a/dev/archery/archery/docker/tests/test_docker.py b/dev/archery/archery/docker/tests/test_docker.py index 899a0449e1a..bc25738becf 100644 --- a/dev/archery/archery/docker/tests/test_docker.py +++ b/dev/archery/archery/docker/tests/test_docker.py @@ -114,6 +114,11 @@ arrow_compose_yml = """ version: '3.5' +x-sccache: &sccache + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + SCCACHE_BUCKET: + x-with-gpus: - ubuntu-cuda @@ -162,6 +167,8 @@ image: org/ubuntu-cpp-cmake32 ubuntu-c-glib: image: org/ubuntu-c-glib + environment: + <<: [*sccache] ubuntu-ruby: image: org/ubuntu-ruby ubuntu-cuda: @@ -529,3 +536,39 @@ def test_listing_images(arrow_compose_path): 'ubuntu-cuda', 'ubuntu-ruby', ] + + +def test_service_info(arrow_compose_path): + compose = DockerCompose(arrow_compose_path) + service = compose.config.raw_config["services"]["conda-cpp"] + assert compose.info(service) == [ + " image: org/conda-cpp", + " build", + " context: .", + " dockerfile: ci/docker/conda-cpp.dockerfile" + ] + + +def test_service_info_filters(arrow_compose_path): + compose = DockerCompose(arrow_compose_path) + service = compose.config.raw_config["services"]["conda-cpp"] + assert compose.info(service, filters="dockerfile") == [ + " dockerfile: ci/docker/conda-cpp.dockerfile" + ] + + +def test_service_info_non_existing_filters(arrow_compose_path): + compose = DockerCompose(arrow_compose_path) + service = compose.config.raw_config["services"]["conda-cpp"] + assert compose.info(service, filters="non-existing") == [] + + +def test_service_info_inherited_env(arrow_compose_path): + compose = DockerCompose(arrow_compose_path) + service = compose.config.raw_config["services"]["ubuntu-c-glib"] + assert compose.info(service, filters="environment") == [ + " environment", + " AWS_ACCESS_KEY_ID: ", + " AWS_SECRET_ACCESS_KEY: ", + " SCCACHE_BUCKET: " + ] diff --git a/dev/merge_arrow_pr.py b/dev/merge_arrow_pr.py index f6ecc7d1ffa..490ee787e33 100755 --- a/dev/merge_arrow_pr.py +++ b/dev/merge_arrow_pr.py @@ -426,7 +426,9 @@ def extract_co_authors(commit): commit_title = f'{self.title} (#{self.number})' commit_message_chunks = [] if self.body is not None: - commit_message_chunks.append(self.body) + # avoid github user name references by inserting a space after @ + body = re.sub(r"@(\w+)", "@ \\1", self.body) + commit_message_chunks.append(body) committer_name = run_cmd("git config --get user.name").strip() committer_email = run_cmd("git config --get user.email").strip() diff --git a/dev/release/post-08-docs.sh b/dev/release/post-08-docs.sh index ad74dbce8d0..0c05cf2192c 100755 --- a/dev/release/post-08-docs.sh +++ b/dev/release/post-08-docs.sh @@ -81,7 +81,8 @@ tar xvf docs.tar.gz rm -f docs.tar.gz git checkout docs/c_glib/index.html if [ "$is_major_release" = "yes" ] ; then - mv docs_temp docs/${previous_version} + previous_series=${previous_version%.*} + mv docs_temp docs/${previous_series} fi git add docs git commit -m "[Website] Update documentations for ${version}" diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index bdb666fd658..14d48b1a615 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -141,6 +141,7 @@ go/arrow/unionmode_string.go go/arrow/compute/go.sum go/arrow/compute/datumkind_string.go go/arrow/compute/funckind_string.go +go/arrow/compute/internal/kernels/_lib/vendored/* go/*.tmpldata go/*.s go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index a9681af5287..902e4ec7134 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -213,6 +213,7 @@ test_apt() { ;; esac if ! docker run --rm -v "${ARROW_DIR}":/arrow:delegated \ + --security-opt="seccomp=unconfined" \ "${target}" \ /arrow/dev/release/verify-apt.sh \ "${VERSION}" \ @@ -1067,11 +1068,12 @@ test_macos_wheels() { # the interpreter should be installed from python.org: # https://www.python.org/ftp/python/3.9.6/python-3.9.6-macosx10.9.pkg if [ "$(uname -m)" = "arm64" ]; then - for pyver in "3.9 3.10"; do + for pyver in 3.9 3.10; do local python="/Library/Frameworks/Python.framework/Versions/${pyver}/bin/python${pyver}" # create and activate a virtualenv for testing as arm64 for arch in "arm64" "x86_64"; do + show_header "Testing Python ${pyver} universal2 wheel on ${arch}" VENV_ENV=wheel-${pyver}-universal2-${arch} PYTHON=${python} maybe_setup_virtualenv || continue # install pyarrow's universal2 wheel pip install pyarrow-${VERSION}-cp${pyver/.}-cp${pyver/.}-macosx_11_0_universal2.whl diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat index 02de305eaaa..21e2ae714ed 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat @@ -31,7 +31,7 @@ cmake -G "Ninja" ^ -DARROW_HDFS:BOOL=ON ^ -DARROW_JSON:BOOL=ON ^ -DARROW_MIMALLOC:BOOL=ON ^ - -DARROW_ORC:BOOL=ON ^ + -DARROW_ORC:BOOL=OFF ^ -DARROW_PACKAGE_PREFIX="%LIBRARY_PREFIX%" ^ -DARROW_PARQUET:BOOL=ON ^ -DARROW_S3:BOOL=ON ^ diff --git a/dev/tasks/linux-packages/github.linux.amd64.yml b/dev/tasks/linux-packages/github.linux.amd64.yml index f252a081d67..d6488d5e714 100644 --- a/dev/tasks/linux-packages/github.linux.amd64.yml +++ b/dev/tasks/linux-packages/github.linux.amd64.yml @@ -44,7 +44,6 @@ jobs: rake version:update rake docker:pull || : rake --trace {{ task_namespace }}:build BUILD_DIR=build - sudo rm -rf */*/build popd env: APT_TARGETS: {{ target }} @@ -103,5 +102,5 @@ jobs: ARROW_VERSION: {{ arrow.version }} YUM_TARGETS: {{ target }} - {% set patterns = upload_extensions | format_all("arrow/dev/tasks/linux-packages/**/*{}") %} + {% set patterns = upload_extensions | format_all("arrow/dev/tasks/linux-packages/*/*/repositories/**/*{}") %} {{ macros.github_upload_releases(patterns)|indent }} diff --git a/dev/tasks/linux-packages/travis.linux.arm64.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml index bc2311a33d6..f3ec4f1de2b 100644 --- a/dev/tasks/linux-packages/travis.linux.arm64.yml +++ b/dev/tasks/linux-packages/travis.linux.arm64.yml @@ -160,5 +160,5 @@ script: - popd after_success: - {% set patterns = upload_extensions | format_all("arrow/dev/tasks/linux-packages/**/*{}") %} + {% set patterns = upload_extensions | format_all("arrow/dev/tasks/linux-packages/*/*/repositories/**/*{}") %} {{ macros.travis_upload_releases(patterns) }} diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 3bec472bcf6..bd3358e0733 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -269,7 +269,7 @@ on: rm -f apache-arrow*.rb.bak {% endmacro %} -{%- macro github_change_r_pkg_version(is_fork, version = '\\2.\'\"$(date +%Y%m%d)\"\'' ) -%} +{%- macro github_change_r_pkg_version(is_fork, version) -%} - name: Modify version shell: bash run: | @@ -339,12 +339,16 @@ on: # tree not available in git-bash on windows run: | ls -R repo - - name: Add dev repo to .Rprofile + - name: Add repos to .Rprofile shell: Rscript {0} run: | - str <- paste0("options(arrow.dev_repo ='file://", getwd(), "/repo' )") - print(str) profile_path <- file.path(getwd(), ".Rprofile") + repo <- paste0("file://", getwd(), "/repo") + str <- paste0("options(arrow.repo = '", repo, "' )") + print(str) + write(str, file = profile_path, append = TRUE) + str <- paste0("options(arrow.dev_repo = '", repo, "' )") + print(str) write(str, file = profile_path, append = TRUE) # Set envvar for later steps by appending to $GITHUB_ENV write(paste0("R_PROFILE_USER=", profile_path), file = Sys.getenv("GITHUB_ENV"), append = TRUE) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 222dbab3a08..e53579f7416 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -17,10 +17,6 @@ {% import 'macros.jinja' as macros with context %} -# This allows us to set a custom version via param: -# crossbow submit --param custom_version=8.5.3 r-binary-packages -# if the param is unset defaults to the usual Ymd naming scheme -{% set package_version = custom_version|replace("Unset", "\\2.\'\"$(date +%Y%m%d)\"\'") %} {% set is_fork = macros.is_fork %} {{ macros.github_header() }} @@ -35,7 +31,7 @@ jobs: pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} steps: {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, package_version)|indent }} + {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} - name: Save Version id: save-version shell: bash @@ -163,7 +159,7 @@ jobs: rig default {{ '${{ matrix.r_version.r }}' }}$rig_arch rig system setup-user-lib - rig system add-pak + rig system add-pak {{ macros.github_setup_local_r_repo(false, true)|indent }} - name: Prepare Dependency Installation @@ -275,18 +271,13 @@ jobs: ARROW_R_DEV: "TRUE" LIBARROW_BUILD: "FALSE" LIBARROW_BINARY: {{ '${{ matrix.config.libarrow_binary }}' }} - DEVTOOLSET_VERSION: {{ '${{ matrix.config.devtoolset }}' }} shell: bash run: | - if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then - # enable the devtoolset version to use it - source /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable - fi Rscript -e ' {{ macros.github_test_r_src_pkg()|indent(8) }} ' - name: Upload binary artifact - if: matrix.config.devtoolset + if: matrix.config.devtoolset uses: actions/upload-artifact@v3 with: name: r-pkg_centos7 @@ -307,11 +298,11 @@ jobs: pkg <- pkg[[1]] warning("Multiple packages found! Using first one.") } - + # Install dependencies from RSPM install.packages("arrow", repos = "https://packagemanager.rstudio.com/all/__linux__/centos7/latest") remove.packages("arrow") - + install.packages(pkg) library(arrow) read_parquet(system.file("v0.7.1.parquet", package = "arrow")) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 81a2c24bba1..bdf53ff1dac 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -957,17 +957,17 @@ tasks: params: custom_version: Unset artifacts: - - r-lib__libarrow__bin__windows__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__centos-7__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__ubuntu-18.04__arrow-[0-9\.]+\.zip - - r-lib__libarrow__bin__ubuntu-22.04__arrow-[0-9\.]+\.zip - - r-pkg__bin__windows__contrib__4.1__arrow_[0-9\.]+\.zip - - r-pkg__bin__windows__contrib__4.2__arrow_[0-9\.]+\.zip - - r-pkg__bin__macosx__contrib__4.1__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__contrib__4.2__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_[0-9\.]+\.tgz - - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_[0-9\.]+\.tgz - - r-pkg__src__contrib__arrow_[0-9\.]+\.tar\.gz + - r-lib__libarrow__bin__windows__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__centos-7__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__ubuntu-18.04__arrow-{no_rc_r_version}\.zip + - r-lib__libarrow__bin__ubuntu-22.04__arrow-{no_rc_r_version}\.zip + - r-pkg__bin__windows__contrib__4.1__arrow_{no_rc_r_version}\.zip + - r-pkg__bin__windows__contrib__4.2__arrow_{no_rc_r_version}\.zip + - r-pkg__bin__macosx__contrib__4.1__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__contrib__4.2__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__big-sur-arm64__contrib__4.1__arrow_{no_rc_r_version}\.tgz + - r-pkg__bin__macosx__big-sur-arm64__contrib__4.2__arrow_{no_rc_r_version}\.tgz + - r-pkg__src__contrib__arrow_{no_rc_r_version}\.tar\.gz ########################### Release verification ############################ diff --git a/docker-compose.yml b/docker-compose.yml index 1c3813757fa..86e4c9fd61b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1068,7 +1068,7 @@ services: ["/arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_build.sh /arrow /build && pip install -e /arrow/dev/archery[numpydoc] && - archery numpydoc --allow-rule PR01,PR10 && + archery numpydoc --allow-rule PR01,PR03,PR10 && /arrow/ci/scripts/python_test.sh /arrow"] conda-python-dask: diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst index fc48b2d65ec..74737cb7496 100644 --- a/docs/source/developers/python.rst +++ b/docs/source/developers/python.rst @@ -198,7 +198,7 @@ dependencies for Arrow C++ and PyArrow as pre-built binaries, which can make Arrow development easier and faster. Let's create a conda environment with all the C++ build and Python dependencies -from conda-forge, targeting development for Python 3.9: +from conda-forge, targeting development for Python 3.10: On Linux and macOS: @@ -210,7 +210,7 @@ On Linux and macOS: --file arrow/ci/conda_env_python.txt \ --file arrow/ci/conda_env_gandiva.txt \ compilers \ - python=3.9 \ + python=3.10 \ pandas As of January 2019, the ``compilers`` package is needed on many Linux @@ -495,23 +495,20 @@ First, starting from a fresh clone of Apache Arrow: --file arrow\ci\conda_env_cpp.txt ^ --file arrow\ci\conda_env_python.txt ^ --file arrow\ci\conda_env_gandiva.txt ^ - python=3.9 + python=3.10 $ conda activate pyarrow-dev Now, we build and install Arrow C++ libraries. -We set a number of environment variables: - -- the path of the installation directory of the Arrow C++ libraries as - ``ARROW_HOME`` -- add the path of installed DLL libraries to ``PATH`` -- and the CMake generator to be used as ``PYARROW_CMAKE_GENERATOR`` +We set the path of the installation directory of the Arrow C++ libraries as +``ARROW_HOME``. When using a conda environment, Arrow C++ is installed +in the environment directory, which path is saved in the +`CONDA_PREFIX `_ +environment variable. .. code-block:: - $ set ARROW_HOME=%cd%\arrow-dist - $ set PATH=%ARROW_HOME%\bin;%PATH% - $ set PYARROW_CMAKE_GENERATOR=Visual Studio 15 2017 Win64 + $ set ARROW_HOME=%CONDA_PREFIX%\Library Let's configure, build and install the Arrow C++ libraries: @@ -519,7 +516,7 @@ Let's configure, build and install the Arrow C++ libraries: $ mkdir arrow\cpp\build $ pushd arrow\cpp\build - $ cmake -G "%PYARROW_CMAKE_GENERATOR%" ^ + $ cmake -G "Ninja" ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ -DCMAKE_UNITY_BUILD=ON ^ -DARROW_COMPUTE=ON ^ @@ -535,7 +532,7 @@ Let's configure, build and install the Arrow C++ libraries: -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ .. - $ cmake --build . --target INSTALL --config Release + $ cmake --build . --target install --config Release $ popd Now, we can build pyarrow: @@ -572,10 +569,6 @@ Then run the unit tests with: the Python extension. This is recommended for development as it allows the C++ libraries to be re-built separately. - As a consequence however, ``python setup.py install`` will also not install - the Arrow C++ libraries. Therefore, to use ``pyarrow`` in python, ``PATH`` - must contain the directory with the Arrow .dll-files. - If you want to bundle the Arrow C++ libraries with ``pyarrow``, add the ``--bundle-arrow-cpp`` option when building: @@ -586,56 +579,10 @@ Then run the unit tests with: Important: If you combine ``--bundle-arrow-cpp`` with ``--inplace`` the Arrow C++ libraries get copied to the source tree and are not cleared by ``python setup.py clean``. They remain in place and will take precedence - over any later Arrow C++ libraries contained in ``PATH``. This can lead to + over any later Arrow C++ libraries contained in ``CONDA_PREFIX``. This can lead to incompatibilities when ``pyarrow`` is later built without ``--bundle-arrow-cpp``. -Running C++ unit tests for Python integration ---------------------------------------------- - -Running C++ unit tests should not be necessary for most developers. If you do -want to run them, you need to pass ``-DARROW_BUILD_TESTS=ON`` during -configuration of the Arrow C++ library build: - -.. code-block:: - - $ mkdir arrow\cpp\build - $ pushd arrow\cpp\build - $ cmake -G "%PYARROW_CMAKE_GENERATOR%" ^ - -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_COMPUTE=ON ^ - -DARROW_CSV=ON ^ - -DARROW_CXXFLAGS="/WX /MP" ^ - -DARROW_DATASET=ON ^ - -DARROW_FILESYSTEM=ON ^ - -DARROW_HDFS=ON ^ - -DARROW_JSON=ON ^ - -DARROW_PARQUET=ON ^ - .. - $ cmake --build . --target INSTALL --config Release - $ popd - -Getting ``arrow-python-test.exe`` (C++ unit tests for python integration) to -run is a bit tricky because your ``%PYTHONHOME%`` must be configured to point -to the active conda environment: - -.. code-block:: - - $ set PYTHONHOME=%CONDA_PREFIX% - $ pushd arrow\cpp\build\release\Release - $ arrow-python-test.exe - $ popd - -To run all tests of the Arrow C++ library, you can also run ``ctest``: - -.. code-block:: - - $ set PYTHONHOME=%CONDA_PREFIX% - $ pushd arrow\cpp\build - $ ctest - $ popd - Caveats ------- diff --git a/docs/source/java/flight_sql.rst b/docs/source/java/flight_sql.rst new file mode 100644 index 00000000000..dbf97238d4c --- /dev/null +++ b/docs/source/java/flight_sql.rst @@ -0,0 +1,32 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +================ +Arrow Flight SQL +================ + +Arrow Flight SQL is an RPC framework for efficient transfer of Arrow data +over the network. + +.. seealso:: + + :doc:`Flight SQL protocol documentation <../format/FlightSql>` + Documentation of the Flight SQL protocol. + +For usage information, see the `API documentation`_. + +.. _API documentation: https://arrow.apache.org/docs/java/reference/org/apache/arrow/flight/sql/package-summary.html diff --git a/docs/source/java/flight_sql_jdbc_driver.rst b/docs/source/java/flight_sql_jdbc_driver.rst new file mode 100644 index 00000000000..65b1a7162f4 --- /dev/null +++ b/docs/source/java/flight_sql_jdbc_driver.rst @@ -0,0 +1,128 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +============================ +Arrow Flight SQL JDBC Driver +============================ + +The Flight SQL JDBC driver is a JDBC driver implementation that uses +the :doc:`Flight SQL protocol <../format/FlightSql>` under the hood. +This driver can be used with any database that implements Flight SQL. + +.. contents:: + +Installation and Requirements +============================= + +The driver is compatible with JDK 8+. On JDK 9+, the following JVM +parameter is required: + +.. code-block:: shell + + java --add-opens=java.base/java.nio=ALL-UNNAMED ... + +To add a dependency via Maven, use a ``pom.xml`` like the following: + +.. code-block:: xml + + + + 4.0.0 + org.example + demo + 1.0-SNAPSHOT + + 10.0.0 + + + + org.apache.arrow + flight-sql-jdbc-driver + ${arrow.version} + + + + +Connecting to a Database +======================== + +The URI format is as follows:: + + jdbc:arrow-flight-sql://HOSTNAME:PORT[/?param1=val1¶m2=val2&...] + +For example, take this URI:: + + jdbc:arrow-flight-sql://localhost:12345/?username=admin&password=pass&useEncryption=1 + +This will connect to a Flight SQL service running on ``localhost`` on +port 12345. It will create a secure, encrypted connection, and +authenticate using the username ``admin`` and the password ``pass``. + +The components of the URI are as follows. + +* The URI scheme must be ``jdbc:arrow-flight-sql://``. +* **HOSTNAME** is the hostname of the Flight SQL service. +* **PORT** is the port of the Flight SQL service. + +Additional options can be passed as query parameters. The supported +parameters are: + +.. list-table:: + :header-rows: 1 + + * - Parameter + - Default + - Description + + * - disableCertificateVerification + - false + - When TLS is enabled, whether to verify the server certificate + + * - password + - null + - The password for user/password authentication + + * - threadPoolSize + - 1 + - The size of an internal thread pool + + * - token + - null + - The token used for token authentication + + * - trustStore + - null + - When TLS is enabled, the path to the certificate store + + * - trustStorePassword + - null + - When TLS is enabled, the password for the certificate store + + * - useEncryption + - false + - Whether to use TLS (the default is an insecure, plaintext + connection) + + * - username + - null + - The username for user/password authentication + + * - useSystemTrustStore + - true + - When TLS is enabled, whether to use the system certificate store diff --git a/docs/source/java/index.rst b/docs/source/java/index.rst index 3c9bde6ba53..a1e924f9c09 100644 --- a/docs/source/java/index.rst +++ b/docs/source/java/index.rst @@ -34,6 +34,8 @@ on the Arrow format and other language bindings see the :doc:`parent documentati ipc algorithm flight + flight_sql + flight_sql_jdbc_driver dataset cdata jdbc diff --git a/go/arrow/bitutil/_lib/bitmap_ops.c b/go/arrow/bitutil/_lib/bitmap_ops.c index 96817b2f2b5..f48b4d4d821 100644 --- a/go/arrow/bitutil/_lib/bitmap_ops.c +++ b/go/arrow/bitutil/_lib/bitmap_ops.c @@ -31,4 +31,16 @@ void FULL_NAME(bitmap_aligned_or)(const uint8_t* left, const uint8_t* right, uin for (int64_t i = 0; i < nbytes; ++i) { out[i] = left[i] | right[i]; } -} \ No newline at end of file +} + +void FULL_NAME(bitmap_aligned_and_not)(const uint8_t* left, const uint8_t* right, uint8_t* out, const int64_t nbytes) { + for (int64_t i = 0; i < nbytes; ++i) { + out[i] = left[i] & ~right[i]; + } +} + +void FULL_NAME(bitmap_aligned_xor)(const uint8_t* left, const uint8_t* right, uint8_t* out, const int64_t nbytes) { + for (int64_t i = 0; i < nbytes; ++i) { + out[i] = left[i] ^ right[i]; + } +} diff --git a/go/arrow/bitutil/_lib/bitmap_ops_avx2_amd64.s b/go/arrow/bitutil/_lib/bitmap_ops_avx2_amd64.s index 69f69d29708..a4010dab55b 100644 --- a/go/arrow/bitutil/_lib/bitmap_ops_avx2_amd64.s +++ b/go/arrow/bitutil/_lib/bitmap_ops_avx2_amd64.s @@ -207,6 +207,204 @@ bitmap_aligned_or_avx2: # @bitmap_aligned_or_avx2 .Lfunc_end1: .size bitmap_aligned_or_avx2, .Lfunc_end1-bitmap_aligned_or_avx2 # -- End function + .globl bitmap_aligned_and_not_avx2 # -- Begin function bitmap_aligned_and_not_avx2 + .p2align 4, 0x90 + .type bitmap_aligned_and_not_avx2,@function +bitmap_aligned_and_not_avx2: # @bitmap_aligned_and_not_avx2 +# %bb.0: + push rbp + mov rbp, rsp + push rbx + and rsp, -8 + test rcx, rcx + jle .LBB2_12 +# %bb.1: + cmp rcx, 127 + ja .LBB2_7 +# %bb.2: + xor r8d, r8d + jmp .LBB2_3 +.LBB2_7: + lea r8, [rdx + rcx] + lea rax, [rdi + rcx] + cmp rax, rdx + seta r11b + lea rax, [rsi + rcx] + cmp r8, rdi + seta bl + cmp rax, rdx + seta r10b + cmp r8, rsi + seta r9b + xor r8d, r8d + test r11b, bl + jne .LBB2_3 +# %bb.8: + and r10b, r9b + jne .LBB2_3 +# %bb.9: + mov r8, rcx + and r8, -128 + xor eax, eax + .p2align 4, 0x90 +.LBB2_10: # =>This Inner Loop Header: Depth=1 + vmovups ymm0, ymmword ptr [rsi + rax] + vmovups ymm1, ymmword ptr [rsi + rax + 32] + vmovups ymm2, ymmword ptr [rsi + rax + 64] + vmovups ymm3, ymmword ptr [rsi + rax + 96] + vandnps ymm0, ymm0, ymmword ptr [rdi + rax] + vandnps ymm1, ymm1, ymmword ptr [rdi + rax + 32] + vandnps ymm2, ymm2, ymmword ptr [rdi + rax + 64] + vandnps ymm3, ymm3, ymmword ptr [rdi + rax + 96] + vmovups ymmword ptr [rdx + rax], ymm0 + vmovups ymmword ptr [rdx + rax + 32], ymm1 + vmovups ymmword ptr [rdx + rax + 64], ymm2 + vmovups ymmword ptr [rdx + rax + 96], ymm3 + sub rax, -128 + cmp r8, rax + jne .LBB2_10 +# %bb.11: + cmp r8, rcx + je .LBB2_12 +.LBB2_3: + mov r9, r8 + not r9 + test cl, 1 + je .LBB2_5 +# %bb.4: + mov al, byte ptr [rsi + r8] + not al + and al, byte ptr [rdi + r8] + mov byte ptr [rdx + r8], al + or r8, 1 +.LBB2_5: + add r9, rcx + je .LBB2_12 + .p2align 4, 0x90 +.LBB2_6: # =>This Inner Loop Header: Depth=1 + movzx eax, byte ptr [rsi + r8] + not al + and al, byte ptr [rdi + r8] + mov byte ptr [rdx + r8], al + movzx eax, byte ptr [rsi + r8 + 1] + not al + and al, byte ptr [rdi + r8 + 1] + mov byte ptr [rdx + r8 + 1], al + add r8, 2 + cmp rcx, r8 + jne .LBB2_6 +.LBB2_12: + lea rsp, [rbp - 8] + pop rbx + pop rbp + vzeroupper + ret +.Lfunc_end2: + .size bitmap_aligned_and_not_avx2, .Lfunc_end2-bitmap_aligned_and_not_avx2 + # -- End function + .globl bitmap_aligned_xor_avx2 # -- Begin function bitmap_aligned_xor_avx2 + .p2align 4, 0x90 + .type bitmap_aligned_xor_avx2,@function +bitmap_aligned_xor_avx2: # @bitmap_aligned_xor_avx2 +# %bb.0: + push rbp + mov rbp, rsp + push rbx + and rsp, -8 + test rcx, rcx + jle .LBB3_12 +# %bb.1: + cmp rcx, 127 + ja .LBB3_7 +# %bb.2: + xor r10d, r10d + jmp .LBB3_3 +.LBB3_7: + lea r9, [rdx + rcx] + lea rax, [rdi + rcx] + cmp rax, rdx + seta r11b + lea rax, [rsi + rcx] + cmp r9, rdi + seta bl + cmp rax, rdx + seta r8b + cmp r9, rsi + seta r9b + xor r10d, r10d + test r11b, bl + jne .LBB3_3 +# %bb.8: + and r8b, r9b + jne .LBB3_3 +# %bb.9: + mov r10, rcx + and r10, -128 + xor r8d, r8d + .p2align 4, 0x90 +.LBB3_10: # =>This Inner Loop Header: Depth=1 + vmovups ymm0, ymmword ptr [rsi + r8] + vmovups ymm1, ymmword ptr [rsi + r8 + 32] + vmovups ymm2, ymmword ptr [rsi + r8 + 64] + vmovups ymm3, ymmword ptr [rsi + r8 + 96] + vxorps ymm0, ymm0, ymmword ptr [rdi + r8] + vxorps ymm1, ymm1, ymmword ptr [rdi + r8 + 32] + vxorps ymm2, ymm2, ymmword ptr [rdi + r8 + 64] + vxorps ymm3, ymm3, ymmword ptr [rdi + r8 + 96] + vmovups ymmword ptr [rdx + r8], ymm0 + vmovups ymmword ptr [rdx + r8 + 32], ymm1 + vmovups ymmword ptr [rdx + r8 + 64], ymm2 + vmovups ymmword ptr [rdx + r8 + 96], ymm3 + sub r8, -128 + cmp r10, r8 + jne .LBB3_10 +# %bb.11: + cmp r10, rcx + je .LBB3_12 +.LBB3_3: + mov r8, r10 + not r8 + add r8, rcx + mov r9, rcx + and r9, 3 + je .LBB3_5 + .p2align 4, 0x90 +.LBB3_4: # =>This Inner Loop Header: Depth=1 + movzx eax, byte ptr [rsi + r10] + xor al, byte ptr [rdi + r10] + mov byte ptr [rdx + r10], al + add r10, 1 + add r9, -1 + jne .LBB3_4 +.LBB3_5: + cmp r8, 3 + jb .LBB3_12 + .p2align 4, 0x90 +.LBB3_6: # =>This Inner Loop Header: Depth=1 + movzx eax, byte ptr [rsi + r10] + xor al, byte ptr [rdi + r10] + mov byte ptr [rdx + r10], al + movzx eax, byte ptr [rsi + r10 + 1] + xor al, byte ptr [rdi + r10 + 1] + mov byte ptr [rdx + r10 + 1], al + movzx eax, byte ptr [rsi + r10 + 2] + xor al, byte ptr [rdi + r10 + 2] + mov byte ptr [rdx + r10 + 2], al + movzx eax, byte ptr [rsi + r10 + 3] + xor al, byte ptr [rdi + r10 + 3] + mov byte ptr [rdx + r10 + 3], al + add r10, 4 + cmp rcx, r10 + jne .LBB3_6 +.LBB3_12: + lea rsp, [rbp - 8] + pop rbx + pop rbp + vzeroupper + ret +.Lfunc_end3: + .size bitmap_aligned_xor_avx2, .Lfunc_end3-bitmap_aligned_xor_avx2 + # -- End function .ident "Ubuntu clang version 11.1.0-6" .section ".note.GNU-stack","",@progbits .addrsig diff --git a/go/arrow/bitutil/_lib/bitmap_ops_sse4_amd64.s b/go/arrow/bitutil/_lib/bitmap_ops_sse4_amd64.s index 9d028155b72..840c1a623bb 100644 --- a/go/arrow/bitutil/_lib/bitmap_ops_sse4_amd64.s +++ b/go/arrow/bitutil/_lib/bitmap_ops_sse4_amd64.s @@ -267,6 +267,264 @@ bitmap_aligned_or_sse4: # @bitmap_aligned_or_sse4 .Lfunc_end1: .size bitmap_aligned_or_sse4, .Lfunc_end1-bitmap_aligned_or_sse4 # -- End function + .globl bitmap_aligned_and_not_sse4 # -- Begin function bitmap_aligned_and_not_sse4 + .p2align 4, 0x90 + .type bitmap_aligned_and_not_sse4,@function +bitmap_aligned_and_not_sse4: # @bitmap_aligned_and_not_sse4 +# %bb.0: + push rbp + mov rbp, rsp + push rbx + and rsp, -8 + test rcx, rcx + jle .LBB2_16 +# %bb.1: + cmp rcx, 31 + ja .LBB2_7 +# %bb.2: + xor r11d, r11d +.LBB2_3: + mov r8, r11 + not r8 + test cl, 1 + je .LBB2_5 +# %bb.4: + mov al, byte ptr [rsi + r11] + not al + and al, byte ptr [rdi + r11] + mov byte ptr [rdx + r11], al + or r11, 1 +.LBB2_5: + add r8, rcx + je .LBB2_16 + .p2align 4, 0x90 +.LBB2_6: # =>This Inner Loop Header: Depth=1 + movzx eax, byte ptr [rsi + r11] + not al + and al, byte ptr [rdi + r11] + mov byte ptr [rdx + r11], al + movzx eax, byte ptr [rsi + r11 + 1] + not al + and al, byte ptr [rdi + r11 + 1] + mov byte ptr [rdx + r11 + 1], al + add r11, 2 + cmp rcx, r11 + jne .LBB2_6 + jmp .LBB2_16 +.LBB2_7: + lea r9, [rdx + rcx] + lea rax, [rdi + rcx] + cmp rax, rdx + seta r10b + lea rax, [rsi + rcx] + cmp r9, rdi + seta bl + cmp rax, rdx + seta r8b + cmp r9, rsi + seta r9b + xor r11d, r11d + test r10b, bl + jne .LBB2_3 +# %bb.8: + and r8b, r9b + jne .LBB2_3 +# %bb.9: + mov r11, rcx + and r11, -32 + lea rax, [r11 - 32] + mov r9, rax + shr r9, 5 + add r9, 1 + test rax, rax + je .LBB2_10 +# %bb.11: + mov r10, r9 + and r10, -2 + neg r10 + xor r8d, r8d + .p2align 4, 0x90 +.LBB2_12: # =>This Inner Loop Header: Depth=1 + movups xmm0, xmmword ptr [rdi + r8] + movups xmm1, xmmword ptr [rdi + r8 + 16] + movups xmm2, xmmword ptr [rsi + r8] + andnps xmm2, xmm0 + movups xmm0, xmmword ptr [rsi + r8 + 16] + andnps xmm0, xmm1 + movups xmmword ptr [rdx + r8], xmm2 + movups xmmword ptr [rdx + r8 + 16], xmm0 + movups xmm0, xmmword ptr [rdi + r8 + 32] + movups xmm1, xmmword ptr [rdi + r8 + 48] + movups xmm2, xmmword ptr [rsi + r8 + 32] + andnps xmm2, xmm0 + movups xmm0, xmmword ptr [rsi + r8 + 48] + andnps xmm0, xmm1 + movups xmmword ptr [rdx + r8 + 32], xmm2 + movups xmmword ptr [rdx + r8 + 48], xmm0 + add r8, 64 + add r10, 2 + jne .LBB2_12 +# %bb.13: + test r9b, 1 + je .LBB2_15 +.LBB2_14: + movups xmm0, xmmword ptr [rdi + r8] + movups xmm1, xmmword ptr [rdi + r8 + 16] + movups xmm2, xmmword ptr [rsi + r8] + andnps xmm2, xmm0 + movups xmm0, xmmword ptr [rsi + r8 + 16] + andnps xmm0, xmm1 + movups xmmword ptr [rdx + r8], xmm2 + movups xmmword ptr [rdx + r8 + 16], xmm0 +.LBB2_15: + cmp r11, rcx + jne .LBB2_3 +.LBB2_16: + lea rsp, [rbp - 8] + pop rbx + pop rbp + ret +.LBB2_10: + xor r8d, r8d + test r9b, 1 + jne .LBB2_14 + jmp .LBB2_15 +.Lfunc_end2: + .size bitmap_aligned_and_not_sse4, .Lfunc_end2-bitmap_aligned_and_not_sse4 + # -- End function + .globl bitmap_aligned_xor_sse4 # -- Begin function bitmap_aligned_xor_sse4 + .p2align 4, 0x90 + .type bitmap_aligned_xor_sse4,@function +bitmap_aligned_xor_sse4: # @bitmap_aligned_xor_sse4 +# %bb.0: + push rbp + mov rbp, rsp + push rbx + and rsp, -8 + test rcx, rcx + jle .LBB3_16 +# %bb.1: + cmp rcx, 31 + ja .LBB3_7 +# %bb.2: + xor r11d, r11d +.LBB3_3: + mov r8, r11 + not r8 + add r8, rcx + mov r9, rcx + and r9, 3 + je .LBB3_5 + .p2align 4, 0x90 +.LBB3_4: # =>This Inner Loop Header: Depth=1 + movzx eax, byte ptr [rsi + r11] + xor al, byte ptr [rdi + r11] + mov byte ptr [rdx + r11], al + add r11, 1 + add r9, -1 + jne .LBB3_4 +.LBB3_5: + cmp r8, 3 + jb .LBB3_16 + .p2align 4, 0x90 +.LBB3_6: # =>This Inner Loop Header: Depth=1 + movzx eax, byte ptr [rsi + r11] + xor al, byte ptr [rdi + r11] + mov byte ptr [rdx + r11], al + movzx eax, byte ptr [rsi + r11 + 1] + xor al, byte ptr [rdi + r11 + 1] + mov byte ptr [rdx + r11 + 1], al + movzx eax, byte ptr [rsi + r11 + 2] + xor al, byte ptr [rdi + r11 + 2] + mov byte ptr [rdx + r11 + 2], al + movzx eax, byte ptr [rsi + r11 + 3] + xor al, byte ptr [rdi + r11 + 3] + mov byte ptr [rdx + r11 + 3], al + add r11, 4 + cmp rcx, r11 + jne .LBB3_6 + jmp .LBB3_16 +.LBB3_7: + lea r9, [rdx + rcx] + lea rax, [rdi + rcx] + cmp rax, rdx + seta r10b + lea rax, [rsi + rcx] + cmp r9, rdi + seta bl + cmp rax, rdx + seta r8b + cmp r9, rsi + seta r9b + xor r11d, r11d + test r10b, bl + jne .LBB3_3 +# %bb.8: + and r8b, r9b + jne .LBB3_3 +# %bb.9: + mov r11, rcx + and r11, -32 + lea rax, [r11 - 32] + mov r9, rax + shr r9, 5 + add r9, 1 + test rax, rax + je .LBB3_10 +# %bb.11: + mov r10, r9 + and r10, -2 + neg r10 + xor r8d, r8d + .p2align 4, 0x90 +.LBB3_12: # =>This Inner Loop Header: Depth=1 + movups xmm0, xmmword ptr [rdi + r8] + movups xmm1, xmmword ptr [rdi + r8 + 16] + movups xmm2, xmmword ptr [rsi + r8] + xorps xmm2, xmm0 + movups xmm0, xmmword ptr [rsi + r8 + 16] + xorps xmm0, xmm1 + movups xmmword ptr [rdx + r8], xmm2 + movups xmmword ptr [rdx + r8 + 16], xmm0 + movups xmm0, xmmword ptr [rdi + r8 + 32] + movups xmm1, xmmword ptr [rdi + r8 + 48] + movups xmm2, xmmword ptr [rsi + r8 + 32] + xorps xmm2, xmm0 + movups xmm0, xmmword ptr [rsi + r8 + 48] + xorps xmm0, xmm1 + movups xmmword ptr [rdx + r8 + 32], xmm2 + movups xmmword ptr [rdx + r8 + 48], xmm0 + add r8, 64 + add r10, 2 + jne .LBB3_12 +# %bb.13: + test r9b, 1 + je .LBB3_15 +.LBB3_14: + movups xmm0, xmmword ptr [rdi + r8] + movups xmm1, xmmword ptr [rdi + r8 + 16] + movups xmm2, xmmword ptr [rsi + r8] + xorps xmm2, xmm0 + movups xmm0, xmmword ptr [rsi + r8 + 16] + xorps xmm0, xmm1 + movups xmmword ptr [rdx + r8], xmm2 + movups xmmword ptr [rdx + r8 + 16], xmm0 +.LBB3_15: + cmp r11, rcx + jne .LBB3_3 +.LBB3_16: + lea rsp, [rbp - 8] + pop rbx + pop rbp + ret +.LBB3_10: + xor r8d, r8d + test r9b, 1 + jne .LBB3_14 + jmp .LBB3_15 +.Lfunc_end3: + .size bitmap_aligned_xor_sse4, .Lfunc_end3-bitmap_aligned_xor_sse4 + # -- End function .ident "Ubuntu clang version 11.1.0-6" .section ".note.GNU-stack","",@progbits .addrsig diff --git a/go/arrow/bitutil/bitmap_ops.go b/go/arrow/bitutil/bitmap_ops.go index 62322b04b9d..7db750a6dd9 100644 --- a/go/arrow/bitutil/bitmap_ops.go +++ b/go/arrow/bitutil/bitmap_ops.go @@ -39,6 +39,29 @@ func alignedBitAndGo(left, right, out []byte) { } } +func alignedBitAndNotGo(left, right, out []byte) { + var ( + nbytes = len(out) + i = 0 + ) + if nbytes > uint64SizeBytes { + // case where we have enough bytes to operate on words + leftWords := bytesToUint64(left[i:]) + rightWords := bytesToUint64(right[i:]) + outWords := bytesToUint64(out[i:]) + + for w := range outWords { + outWords[w] = leftWords[w] &^ rightWords[w] + } + + i += len(outWords) * uint64SizeBytes + } + // grab any remaining bytes that were fewer than a word + for ; i < nbytes; i++ { + out[i] = left[i] &^ right[i] + } +} + func alignedBitOrGo(left, right, out []byte) { var ( nbytes = len(out) @@ -61,3 +84,26 @@ func alignedBitOrGo(left, right, out []byte) { out[i] = left[i] | right[i] } } + +func alignedBitXorGo(left, right, out []byte) { + var ( + nbytes = len(out) + i = 0 + ) + if nbytes > uint64SizeBytes { + // case where we have enough bytes to operate on words + leftWords := bytesToUint64(left[i:]) + rightWords := bytesToUint64(right[i:]) + outWords := bytesToUint64(out[i:]) + + for w := range outWords { + outWords[w] = leftWords[w] ^ rightWords[w] + } + + i += len(outWords) * uint64SizeBytes + } + // grab any remaining bytes that were fewer than a word + for ; i < nbytes; i++ { + out[i] = left[i] ^ right[i] + } +} diff --git a/go/arrow/bitutil/bitmap_ops_amd64.go b/go/arrow/bitutil/bitmap_ops_amd64.go index 9aa5a6dd56b..ad0fd674ab9 100644 --- a/go/arrow/bitutil/bitmap_ops_amd64.go +++ b/go/arrow/bitutil/bitmap_ops_amd64.go @@ -25,11 +25,17 @@ func init() { if cpu.X86.HasAVX2 { bitAndOp.opAligned = bitmapAlignedAndAVX2 bitOrOp.opAligned = bitmapAlignedOrAVX2 + bitAndNotOp.opAligned = bitmapAlignedAndNotAVX2 + bitXorOp.opAligned = bitmapAlignedXorAVX2 } else if cpu.X86.HasSSE42 { bitAndOp.opAligned = bitmapAlignedAndSSE4 bitOrOp.opAligned = bitmapAlignedOrSSE4 + bitAndNotOp.opAligned = bitmapAlignedAndNotSSE4 + bitXorOp.opAligned = bitmapAlignedXorSSE4 } else { bitAndOp.opAligned = alignedBitAndGo bitOrOp.opAligned = alignedBitOrGo + bitAndNotOp.opAligned = alignedBitAndNotGo + bitXorOp.opAligned = alignedBitXorGo } } diff --git a/go/arrow/bitutil/bitmap_ops_arm64.go b/go/arrow/bitutil/bitmap_ops_arm64.go index 86c47639a9e..28d95d84ade 100644 --- a/go/arrow/bitutil/bitmap_ops_arm64.go +++ b/go/arrow/bitutil/bitmap_ops_arm64.go @@ -22,4 +22,6 @@ package bitutil func init() { bitAndOp.opAligned = alignedBitAndGo bitOrOp.opAligned = alignedBitOrGo + bitAndNotOp.opAligned = alignedBitAndNotGo + bitXorOp.opAligned = alignedBitXorGo } diff --git a/go/arrow/bitutil/bitmap_ops_avx2_amd64.go b/go/arrow/bitutil/bitmap_ops_avx2_amd64.go index 731b9807b79..1c01bd0f380 100644 --- a/go/arrow/bitutil/bitmap_ops_avx2_amd64.go +++ b/go/arrow/bitutil/bitmap_ops_avx2_amd64.go @@ -36,3 +36,17 @@ func _bitmap_aligned_or_avx2(left, right, out unsafe.Pointer, length int64) func bitmapAlignedOrAVX2(left, right, out []byte) { _bitmap_aligned_or_avx2(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) } + +//go:noescape +func _bitmap_aligned_and_not_avx2(left, right, out unsafe.Pointer, length int64) + +func bitmapAlignedAndNotAVX2(left, right, out []byte) { + _bitmap_aligned_and_not_avx2(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) +} + +//go:noescape +func _bitmap_aligned_xor_avx2(left, right, out unsafe.Pointer, length int64) + +func bitmapAlignedXorAVX2(left, right, out []byte) { + _bitmap_aligned_xor_avx2(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) +} diff --git a/go/arrow/bitutil/bitmap_ops_avx2_amd64.s b/go/arrow/bitutil/bitmap_ops_avx2_amd64.s index 2e2ade89617..00172e86592 100644 --- a/go/arrow/bitutil/bitmap_ops_avx2_amd64.s +++ b/go/arrow/bitutil/bitmap_ops_avx2_amd64.s @@ -190,3 +190,184 @@ LBB1_6: LBB1_12: VZEROUPPER RET + +TEXT ·_bitmap_aligned_and_not_avx2(SB), $0-32 + + MOVQ left+0(FP), DI + MOVQ right+8(FP), SI + MOVQ out+16(FP), DX + MOVQ length+24(FP), CX + + WORD $0x8548; BYTE $0xc9 // test rcx, rcx + JLE LBB2_12 + LONG $0x7ff98348 // cmp rcx, 127 + JA LBB2_7 + WORD $0x3145; BYTE $0xc0 // xor r8d, r8d + JMP LBB2_3 + +LBB2_7: + LONG $0x0a048d4c // lea r8, [rdx + rcx] + LONG $0x0f048d48 // lea rax, [rdi + rcx] + WORD $0x3948; BYTE $0xd0 // cmp rax, rdx + LONG $0xd3970f41 // seta r11b + LONG $0x0e048d48 // lea rax, [rsi + rcx] + WORD $0x3949; BYTE $0xf8 // cmp r8, rdi + WORD $0x970f; BYTE $0xd3 // seta bl + WORD $0x3948; BYTE $0xd0 // cmp rax, rdx + LONG $0xd2970f41 // seta r10b + WORD $0x3949; BYTE $0xf0 // cmp r8, rsi + LONG $0xd1970f41 // seta r9b + WORD $0x3145; BYTE $0xc0 // xor r8d, r8d + WORD $0x8441; BYTE $0xdb // test r11b, bl + JNE LBB2_3 + WORD $0x2045; BYTE $0xca // and r10b, r9b + JNE LBB2_3 + WORD $0x8949; BYTE $0xc8 // mov r8, rcx + LONG $0x80e08349 // and r8, -128 + WORD $0xc031 // xor eax, eax + +LBB2_10: + LONG $0x0410fcc5; BYTE $0x06 // vmovups ymm0, yword [rsi + rax] + LONG $0x4c10fcc5; WORD $0x2006 // vmovups ymm1, yword [rsi + rax + 32] + LONG $0x5410fcc5; WORD $0x4006 // vmovups ymm2, yword [rsi + rax + 64] + LONG $0x5c10fcc5; WORD $0x6006 // vmovups ymm3, yword [rsi + rax + 96] + LONG $0x0455fcc5; BYTE $0x07 // vandnps ymm0, ymm0, yword [rdi + rax] + LONG $0x4c55f4c5; WORD $0x2007 // vandnps ymm1, ymm1, yword [rdi + rax + 32] + LONG $0x5455ecc5; WORD $0x4007 // vandnps ymm2, ymm2, yword [rdi + rax + 64] + LONG $0x5c55e4c5; WORD $0x6007 // vandnps ymm3, ymm3, yword [rdi + rax + 96] + LONG $0x0411fcc5; BYTE $0x02 // vmovups yword [rdx + rax], ymm0 + LONG $0x4c11fcc5; WORD $0x2002 // vmovups yword [rdx + rax + 32], ymm1 + LONG $0x5411fcc5; WORD $0x4002 // vmovups yword [rdx + rax + 64], ymm2 + LONG $0x5c11fcc5; WORD $0x6002 // vmovups yword [rdx + rax + 96], ymm3 + LONG $0x80e88348 // sub rax, -128 + WORD $0x3949; BYTE $0xc0 // cmp r8, rax + JNE LBB2_10 + WORD $0x3949; BYTE $0xc8 // cmp r8, rcx + JE LBB2_12 + +LBB2_3: + WORD $0x894d; BYTE $0xc1 // mov r9, r8 + WORD $0xf749; BYTE $0xd1 // not r9 + WORD $0xc1f6; BYTE $0x01 // test cl, 1 + JE LBB2_5 + LONG $0x06048a42 // mov al, byte [rsi + r8] + WORD $0xd0f6 // not al + LONG $0x07042242 // and al, byte [rdi + r8] + LONG $0x02048842 // mov byte [rdx + r8], al + LONG $0x01c88349 // or r8, 1 + +LBB2_5: + WORD $0x0149; BYTE $0xc9 // add r9, rcx + JE LBB2_12 + +LBB2_6: + LONG $0x04b60f42; BYTE $0x06 // movzx eax, byte [rsi + r8] + WORD $0xd0f6 // not al + LONG $0x07042242 // and al, byte [rdi + r8] + LONG $0x02048842 // mov byte [rdx + r8], al + LONG $0x44b60f42; WORD $0x0106 // movzx eax, byte [rsi + r8 + 1] + WORD $0xd0f6 // not al + LONG $0x07442242; BYTE $0x01 // and al, byte [rdi + r8 + 1] + LONG $0x02448842; BYTE $0x01 // mov byte [rdx + r8 + 1], al + LONG $0x02c08349 // add r8, 2 + WORD $0x394c; BYTE $0xc1 // cmp rcx, r8 + JNE LBB2_6 + +LBB2_12: + VZEROUPPER + RET + +TEXT ·_bitmap_aligned_xor_avx2(SB), $0-32 + + MOVQ left+0(FP), DI + MOVQ right+8(FP), SI + MOVQ out+16(FP), DX + MOVQ length+24(FP), CX + + WORD $0x8548; BYTE $0xc9 // test rcx, rcx + JLE LBB3_12 + LONG $0x7ff98348 // cmp rcx, 127 + JA LBB3_7 + WORD $0x3145; BYTE $0xd2 // xor r10d, r10d + JMP LBB3_3 + +LBB3_7: + LONG $0x0a0c8d4c // lea r9, [rdx + rcx] + LONG $0x0f048d48 // lea rax, [rdi + rcx] + WORD $0x3948; BYTE $0xd0 // cmp rax, rdx + LONG $0xd3970f41 // seta r11b + LONG $0x0e048d48 // lea rax, [rsi + rcx] + WORD $0x3949; BYTE $0xf9 // cmp r9, rdi + WORD $0x970f; BYTE $0xd3 // seta bl + WORD $0x3948; BYTE $0xd0 // cmp rax, rdx + LONG $0xd0970f41 // seta r8b + WORD $0x3949; BYTE $0xf1 // cmp r9, rsi + LONG $0xd1970f41 // seta r9b + WORD $0x3145; BYTE $0xd2 // xor r10d, r10d + WORD $0x8441; BYTE $0xdb // test r11b, bl + JNE LBB3_3 + WORD $0x2045; BYTE $0xc8 // and r8b, r9b + JNE LBB3_3 + WORD $0x8949; BYTE $0xca // mov r10, rcx + LONG $0x80e28349 // and r10, -128 + WORD $0x3145; BYTE $0xc0 // xor r8d, r8d + +LBB3_10: + LONG $0x107ca1c4; WORD $0x0604 // vmovups ymm0, yword [rsi + r8] + LONG $0x107ca1c4; WORD $0x064c; BYTE $0x20 // vmovups ymm1, yword [rsi + r8 + 32] + LONG $0x107ca1c4; WORD $0x0654; BYTE $0x40 // vmovups ymm2, yword [rsi + r8 + 64] + LONG $0x107ca1c4; WORD $0x065c; BYTE $0x60 // vmovups ymm3, yword [rsi + r8 + 96] + LONG $0x577ca1c4; WORD $0x0704 // vxorps ymm0, ymm0, yword [rdi + r8] + LONG $0x5774a1c4; WORD $0x074c; BYTE $0x20 // vxorps ymm1, ymm1, yword [rdi + r8 + 32] + LONG $0x576ca1c4; WORD $0x0754; BYTE $0x40 // vxorps ymm2, ymm2, yword [rdi + r8 + 64] + LONG $0x5764a1c4; WORD $0x075c; BYTE $0x60 // vxorps ymm3, ymm3, yword [rdi + r8 + 96] + LONG $0x117ca1c4; WORD $0x0204 // vmovups yword [rdx + r8], ymm0 + LONG $0x117ca1c4; WORD $0x024c; BYTE $0x20 // vmovups yword [rdx + r8 + 32], ymm1 + LONG $0x117ca1c4; WORD $0x0254; BYTE $0x40 // vmovups yword [rdx + r8 + 64], ymm2 + LONG $0x117ca1c4; WORD $0x025c; BYTE $0x60 // vmovups yword [rdx + r8 + 96], ymm3 + LONG $0x80e88349 // sub r8, -128 + WORD $0x394d; BYTE $0xc2 // cmp r10, r8 + JNE LBB3_10 + WORD $0x3949; BYTE $0xca // cmp r10, rcx + JE LBB3_12 + +LBB3_3: + WORD $0x894d; BYTE $0xd0 // mov r8, r10 + WORD $0xf749; BYTE $0xd0 // not r8 + WORD $0x0149; BYTE $0xc8 // add r8, rcx + WORD $0x8949; BYTE $0xc9 // mov r9, rcx + LONG $0x03e18349 // and r9, 3 + JE LBB3_5 + +LBB3_4: + LONG $0x04b60f42; BYTE $0x16 // movzx eax, byte [rsi + r10] + LONG $0x17043242 // xor al, byte [rdi + r10] + LONG $0x12048842 // mov byte [rdx + r10], al + LONG $0x01c28349 // add r10, 1 + LONG $0xffc18349 // add r9, -1 + JNE LBB3_4 + +LBB3_5: + LONG $0x03f88349 // cmp r8, 3 + JB LBB3_12 + +LBB3_6: + LONG $0x04b60f42; BYTE $0x16 // movzx eax, byte [rsi + r10] + LONG $0x17043242 // xor al, byte [rdi + r10] + LONG $0x12048842 // mov byte [rdx + r10], al + LONG $0x44b60f42; WORD $0x0116 // movzx eax, byte [rsi + r10 + 1] + LONG $0x17443242; BYTE $0x01 // xor al, byte [rdi + r10 + 1] + LONG $0x12448842; BYTE $0x01 // mov byte [rdx + r10 + 1], al + LONG $0x44b60f42; WORD $0x0216 // movzx eax, byte [rsi + r10 + 2] + LONG $0x17443242; BYTE $0x02 // xor al, byte [rdi + r10 + 2] + LONG $0x12448842; BYTE $0x02 // mov byte [rdx + r10 + 2], al + LONG $0x44b60f42; WORD $0x0316 // movzx eax, byte [rsi + r10 + 3] + LONG $0x17443242; BYTE $0x03 // xor al, byte [rdi + r10 + 3] + LONG $0x12448842; BYTE $0x03 // mov byte [rdx + r10 + 3], al + LONG $0x04c28349 // add r10, 4 + WORD $0x394c; BYTE $0xd1 // cmp rcx, r10 + JNE LBB3_6 + +LBB3_12: + VZEROUPPER + RET diff --git a/go/arrow/bitutil/bitmap_ops_noasm.go b/go/arrow/bitutil/bitmap_ops_noasm.go index 785531c1c23..e25347791fe 100644 --- a/go/arrow/bitutil/bitmap_ops_noasm.go +++ b/go/arrow/bitutil/bitmap_ops_noasm.go @@ -22,4 +22,6 @@ package bitutil func init() { bitAndOp.opAligned = alignedBitAndGo bitOrOp.opAligned = alignedBitOrGo + bitAndNotOp.opAligned = alignedBitAndNotGo + bitXorOp.opAligned = alignedBitXorGo } diff --git a/go/arrow/bitutil/bitmap_ops_ppc64le.go b/go/arrow/bitutil/bitmap_ops_ppc64le.go index 86c47639a9e..28d95d84ade 100644 --- a/go/arrow/bitutil/bitmap_ops_ppc64le.go +++ b/go/arrow/bitutil/bitmap_ops_ppc64le.go @@ -22,4 +22,6 @@ package bitutil func init() { bitAndOp.opAligned = alignedBitAndGo bitOrOp.opAligned = alignedBitOrGo + bitAndNotOp.opAligned = alignedBitAndNotGo + bitXorOp.opAligned = alignedBitXorGo } diff --git a/go/arrow/bitutil/bitmap_ops_s390x.go b/go/arrow/bitutil/bitmap_ops_s390x.go index 86c47639a9e..28d95d84ade 100644 --- a/go/arrow/bitutil/bitmap_ops_s390x.go +++ b/go/arrow/bitutil/bitmap_ops_s390x.go @@ -22,4 +22,6 @@ package bitutil func init() { bitAndOp.opAligned = alignedBitAndGo bitOrOp.opAligned = alignedBitOrGo + bitAndNotOp.opAligned = alignedBitAndNotGo + bitXorOp.opAligned = alignedBitXorGo } diff --git a/go/arrow/bitutil/bitmap_ops_sse4_amd64.go b/go/arrow/bitutil/bitmap_ops_sse4_amd64.go index 5d1fcf96829..f16bce12bbf 100644 --- a/go/arrow/bitutil/bitmap_ops_sse4_amd64.go +++ b/go/arrow/bitutil/bitmap_ops_sse4_amd64.go @@ -36,3 +36,17 @@ func _bitmap_aligned_or_sse4(left, right, out unsafe.Pointer, length int64) func bitmapAlignedOrSSE4(left, right, out []byte) { _bitmap_aligned_or_sse4(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) } + +//go:noescape +func _bitmap_aligned_and_not_sse4(left, right, out unsafe.Pointer, length int64) + +func bitmapAlignedAndNotSSE4(left, right, out []byte) { + _bitmap_aligned_and_not_sse4(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) +} + +//go:noescape +func _bitmap_aligned_xor_sse4(left, right, out unsafe.Pointer, length int64) + +func bitmapAlignedXorSSE4(left, right, out []byte) { + _bitmap_aligned_xor_sse4(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) +} diff --git a/go/arrow/bitutil/bitmap_ops_sse4_amd64.s b/go/arrow/bitutil/bitmap_ops_sse4_amd64.s index ad81cf63720..c15e186253a 100644 --- a/go/arrow/bitutil/bitmap_ops_sse4_amd64.s +++ b/go/arrow/bitutil/bitmap_ops_sse4_amd64.s @@ -254,3 +254,248 @@ LBB1_10: LONG $0x01c1f641 // test r9b, 1 JNE LBB1_14 JMP LBB1_15 + +TEXT ·_bitmap_aligned_and_not_sse4(SB), $0-32 + + MOVQ left+0(FP), DI + MOVQ right+8(FP), SI + MOVQ out+16(FP), DX + MOVQ length+24(FP), CX + + WORD $0x8548; BYTE $0xc9 // test rcx, rcx + JLE LBB2_16 + LONG $0x1ff98348 // cmp rcx, 31 + JA LBB2_7 + WORD $0x3145; BYTE $0xdb // xor r11d, r11d + +LBB2_3: + WORD $0x894d; BYTE $0xd8 // mov r8, r11 + WORD $0xf749; BYTE $0xd0 // not r8 + WORD $0xc1f6; BYTE $0x01 // test cl, 1 + JE LBB2_5 + LONG $0x1e048a42 // mov al, byte [rsi + r11] + WORD $0xd0f6 // not al + LONG $0x1f042242 // and al, byte [rdi + r11] + LONG $0x1a048842 // mov byte [rdx + r11], al + LONG $0x01cb8349 // or r11, 1 + +LBB2_5: + WORD $0x0149; BYTE $0xc8 // add r8, rcx + JE LBB2_16 + +LBB2_6: + LONG $0x04b60f42; BYTE $0x1e // movzx eax, byte [rsi + r11] + WORD $0xd0f6 // not al + LONG $0x1f042242 // and al, byte [rdi + r11] + LONG $0x1a048842 // mov byte [rdx + r11], al + LONG $0x44b60f42; WORD $0x011e // movzx eax, byte [rsi + r11 + 1] + WORD $0xd0f6 // not al + LONG $0x1f442242; BYTE $0x01 // and al, byte [rdi + r11 + 1] + LONG $0x1a448842; BYTE $0x01 // mov byte [rdx + r11 + 1], al + LONG $0x02c38349 // add r11, 2 + WORD $0x394c; BYTE $0xd9 // cmp rcx, r11 + JNE LBB2_6 + JMP LBB2_16 + +LBB2_7: + LONG $0x0a0c8d4c // lea r9, [rdx + rcx] + LONG $0x0f048d48 // lea rax, [rdi + rcx] + WORD $0x3948; BYTE $0xd0 // cmp rax, rdx + LONG $0xd2970f41 // seta r10b + LONG $0x0e048d48 // lea rax, [rsi + rcx] + WORD $0x3949; BYTE $0xf9 // cmp r9, rdi + WORD $0x970f; BYTE $0xd3 // seta bl + WORD $0x3948; BYTE $0xd0 // cmp rax, rdx + LONG $0xd0970f41 // seta r8b + WORD $0x3949; BYTE $0xf1 // cmp r9, rsi + LONG $0xd1970f41 // seta r9b + WORD $0x3145; BYTE $0xdb // xor r11d, r11d + WORD $0x8441; BYTE $0xda // test r10b, bl + JNE LBB2_3 + WORD $0x2045; BYTE $0xc8 // and r8b, r9b + JNE LBB2_3 + WORD $0x8949; BYTE $0xcb // mov r11, rcx + LONG $0xe0e38349 // and r11, -32 + LONG $0xe0438d49 // lea rax, [r11 - 32] + WORD $0x8949; BYTE $0xc1 // mov r9, rax + LONG $0x05e9c149 // shr r9, 5 + LONG $0x01c18349 // add r9, 1 + WORD $0x8548; BYTE $0xc0 // test rax, rax + JE LBB2_10 + WORD $0x894d; BYTE $0xca // mov r10, r9 + LONG $0xfee28349 // and r10, -2 + WORD $0xf749; BYTE $0xda // neg r10 + WORD $0x3145; BYTE $0xc0 // xor r8d, r8d + +LBB2_12: + LONG $0x04100f42; BYTE $0x07 // movups xmm0, oword [rdi + r8] + LONG $0x4c100f42; WORD $0x1007 // movups xmm1, oword [rdi + r8 + 16] + LONG $0x14100f42; BYTE $0x06 // movups xmm2, oword [rsi + r8] + WORD $0x550f; BYTE $0xd0 // andnps xmm2, xmm0 + LONG $0x44100f42; WORD $0x1006 // movups xmm0, oword [rsi + r8 + 16] + WORD $0x550f; BYTE $0xc1 // andnps xmm0, xmm1 + LONG $0x14110f42; BYTE $0x02 // movups oword [rdx + r8], xmm2 + LONG $0x44110f42; WORD $0x1002 // movups oword [rdx + r8 + 16], xmm0 + LONG $0x44100f42; WORD $0x2007 // movups xmm0, oword [rdi + r8 + 32] + LONG $0x4c100f42; WORD $0x3007 // movups xmm1, oword [rdi + r8 + 48] + LONG $0x54100f42; WORD $0x2006 // movups xmm2, oword [rsi + r8 + 32] + WORD $0x550f; BYTE $0xd0 // andnps xmm2, xmm0 + LONG $0x44100f42; WORD $0x3006 // movups xmm0, oword [rsi + r8 + 48] + WORD $0x550f; BYTE $0xc1 // andnps xmm0, xmm1 + LONG $0x54110f42; WORD $0x2002 // movups oword [rdx + r8 + 32], xmm2 + LONG $0x44110f42; WORD $0x3002 // movups oword [rdx + r8 + 48], xmm0 + LONG $0x40c08349 // add r8, 64 + LONG $0x02c28349 // add r10, 2 + JNE LBB2_12 + LONG $0x01c1f641 // test r9b, 1 + JE LBB2_15 + +LBB2_14: + LONG $0x04100f42; BYTE $0x07 // movups xmm0, oword [rdi + r8] + LONG $0x4c100f42; WORD $0x1007 // movups xmm1, oword [rdi + r8 + 16] + LONG $0x14100f42; BYTE $0x06 // movups xmm2, oword [rsi + r8] + WORD $0x550f; BYTE $0xd0 // andnps xmm2, xmm0 + LONG $0x44100f42; WORD $0x1006 // movups xmm0, oword [rsi + r8 + 16] + WORD $0x550f; BYTE $0xc1 // andnps xmm0, xmm1 + LONG $0x14110f42; BYTE $0x02 // movups oword [rdx + r8], xmm2 + LONG $0x44110f42; WORD $0x1002 // movups oword [rdx + r8 + 16], xmm0 + +LBB2_15: + WORD $0x3949; BYTE $0xcb // cmp r11, rcx + JNE LBB2_3 + +LBB2_16: + RET + +LBB2_10: + WORD $0x3145; BYTE $0xc0 // xor r8d, r8d + LONG $0x01c1f641 // test r9b, 1 + JNE LBB2_14 + JMP LBB2_15 + +TEXT ·_bitmap_aligned_xor_sse4(SB), $0-32 + + MOVQ left+0(FP), DI + MOVQ right+8(FP), SI + MOVQ out+16(FP), DX + MOVQ length+24(FP), CX + + WORD $0x8548; BYTE $0xc9 // test rcx, rcx + JLE LBB3_16 + LONG $0x1ff98348 // cmp rcx, 31 + JA LBB3_7 + WORD $0x3145; BYTE $0xdb // xor r11d, r11d + +LBB3_3: + WORD $0x894d; BYTE $0xd8 // mov r8, r11 + WORD $0xf749; BYTE $0xd0 // not r8 + WORD $0x0149; BYTE $0xc8 // add r8, rcx + WORD $0x8949; BYTE $0xc9 // mov r9, rcx + LONG $0x03e18349 // and r9, 3 + JE LBB3_5 + +LBB3_4: + LONG $0x04b60f42; BYTE $0x1e // movzx eax, byte [rsi + r11] + LONG $0x1f043242 // xor al, byte [rdi + r11] + LONG $0x1a048842 // mov byte [rdx + r11], al + LONG $0x01c38349 // add r11, 1 + LONG $0xffc18349 // add r9, -1 + JNE LBB3_4 + +LBB3_5: + LONG $0x03f88349 // cmp r8, 3 + JB LBB3_16 + +LBB3_6: + LONG $0x04b60f42; BYTE $0x1e // movzx eax, byte [rsi + r11] + LONG $0x1f043242 // xor al, byte [rdi + r11] + LONG $0x1a048842 // mov byte [rdx + r11], al + LONG $0x44b60f42; WORD $0x011e // movzx eax, byte [rsi + r11 + 1] + LONG $0x1f443242; BYTE $0x01 // xor al, byte [rdi + r11 + 1] + LONG $0x1a448842; BYTE $0x01 // mov byte [rdx + r11 + 1], al + LONG $0x44b60f42; WORD $0x021e // movzx eax, byte [rsi + r11 + 2] + LONG $0x1f443242; BYTE $0x02 // xor al, byte [rdi + r11 + 2] + LONG $0x1a448842; BYTE $0x02 // mov byte [rdx + r11 + 2], al + LONG $0x44b60f42; WORD $0x031e // movzx eax, byte [rsi + r11 + 3] + LONG $0x1f443242; BYTE $0x03 // xor al, byte [rdi + r11 + 3] + LONG $0x1a448842; BYTE $0x03 // mov byte [rdx + r11 + 3], al + LONG $0x04c38349 // add r11, 4 + WORD $0x394c; BYTE $0xd9 // cmp rcx, r11 + JNE LBB3_6 + JMP LBB3_16 + +LBB3_7: + LONG $0x0a0c8d4c // lea r9, [rdx + rcx] + LONG $0x0f048d48 // lea rax, [rdi + rcx] + WORD $0x3948; BYTE $0xd0 // cmp rax, rdx + LONG $0xd2970f41 // seta r10b + LONG $0x0e048d48 // lea rax, [rsi + rcx] + WORD $0x3949; BYTE $0xf9 // cmp r9, rdi + WORD $0x970f; BYTE $0xd3 // seta bl + WORD $0x3948; BYTE $0xd0 // cmp rax, rdx + LONG $0xd0970f41 // seta r8b + WORD $0x3949; BYTE $0xf1 // cmp r9, rsi + LONG $0xd1970f41 // seta r9b + WORD $0x3145; BYTE $0xdb // xor r11d, r11d + WORD $0x8441; BYTE $0xda // test r10b, bl + JNE LBB3_3 + WORD $0x2045; BYTE $0xc8 // and r8b, r9b + JNE LBB3_3 + WORD $0x8949; BYTE $0xcb // mov r11, rcx + LONG $0xe0e38349 // and r11, -32 + LONG $0xe0438d49 // lea rax, [r11 - 32] + WORD $0x8949; BYTE $0xc1 // mov r9, rax + LONG $0x05e9c149 // shr r9, 5 + LONG $0x01c18349 // add r9, 1 + WORD $0x8548; BYTE $0xc0 // test rax, rax + JE LBB3_10 + WORD $0x894d; BYTE $0xca // mov r10, r9 + LONG $0xfee28349 // and r10, -2 + WORD $0xf749; BYTE $0xda // neg r10 + WORD $0x3145; BYTE $0xc0 // xor r8d, r8d + +LBB3_12: + LONG $0x04100f42; BYTE $0x07 // movups xmm0, oword [rdi + r8] + LONG $0x4c100f42; WORD $0x1007 // movups xmm1, oword [rdi + r8 + 16] + LONG $0x14100f42; BYTE $0x06 // movups xmm2, oword [rsi + r8] + WORD $0x570f; BYTE $0xd0 // xorps xmm2, xmm0 + LONG $0x44100f42; WORD $0x1006 // movups xmm0, oword [rsi + r8 + 16] + WORD $0x570f; BYTE $0xc1 // xorps xmm0, xmm1 + LONG $0x14110f42; BYTE $0x02 // movups oword [rdx + r8], xmm2 + LONG $0x44110f42; WORD $0x1002 // movups oword [rdx + r8 + 16], xmm0 + LONG $0x44100f42; WORD $0x2007 // movups xmm0, oword [rdi + r8 + 32] + LONG $0x4c100f42; WORD $0x3007 // movups xmm1, oword [rdi + r8 + 48] + LONG $0x54100f42; WORD $0x2006 // movups xmm2, oword [rsi + r8 + 32] + WORD $0x570f; BYTE $0xd0 // xorps xmm2, xmm0 + LONG $0x44100f42; WORD $0x3006 // movups xmm0, oword [rsi + r8 + 48] + WORD $0x570f; BYTE $0xc1 // xorps xmm0, xmm1 + LONG $0x54110f42; WORD $0x2002 // movups oword [rdx + r8 + 32], xmm2 + LONG $0x44110f42; WORD $0x3002 // movups oword [rdx + r8 + 48], xmm0 + LONG $0x40c08349 // add r8, 64 + LONG $0x02c28349 // add r10, 2 + JNE LBB3_12 + LONG $0x01c1f641 // test r9b, 1 + JE LBB3_15 + +LBB3_14: + LONG $0x04100f42; BYTE $0x07 // movups xmm0, oword [rdi + r8] + LONG $0x4c100f42; WORD $0x1007 // movups xmm1, oword [rdi + r8 + 16] + LONG $0x14100f42; BYTE $0x06 // movups xmm2, oword [rsi + r8] + WORD $0x570f; BYTE $0xd0 // xorps xmm2, xmm0 + LONG $0x44100f42; WORD $0x1006 // movups xmm0, oword [rsi + r8 + 16] + WORD $0x570f; BYTE $0xc1 // xorps xmm0, xmm1 + LONG $0x14110f42; BYTE $0x02 // movups oword [rdx + r8], xmm2 + LONG $0x44110f42; WORD $0x1002 // movups oword [rdx + r8 + 16], xmm0 + +LBB3_15: + WORD $0x3949; BYTE $0xcb // cmp r11, rcx + JNE LBB3_3 + +LBB3_16: + RET + +LBB3_10: + WORD $0x3145; BYTE $0xc0 // xor r8d, r8d + LONG $0x01c1f641 // test r9b, 1 + JNE LBB3_14 + JMP LBB3_15 diff --git a/go/arrow/bitutil/bitmaps.go b/go/arrow/bitutil/bitmaps.go index abd1b188a74..c23a1232921 100644 --- a/go/arrow/bitutil/bitmaps.go +++ b/go/arrow/bitutil/bitmaps.go @@ -18,6 +18,7 @@ package bitutil import ( "bytes" + "errors" "math/bits" "unsafe" @@ -374,9 +375,14 @@ func (bm *BitmapWordWriter) PutNextTrailingByte(b byte, validBits int) { } } -// CopyBitmap copies the bitmap indicated by src, starting at bit offset srcOffset, -// and copying length bits into dst, starting at bit offset dstOffset. -func CopyBitmap(src []byte, srcOffset, length int, dst []byte, dstOffset int) { +type transferMode int8 + +const ( + transferCopy transferMode = iota + transferInvert +) + +func transferBitmap(mode transferMode, src []byte, srcOffset, length int, dst []byte, dstOffset int) { if length == 0 { // if there's nothing to write, end early. return @@ -393,12 +399,19 @@ func CopyBitmap(src []byte, srcOffset, length int, dst []byte, dstOffset int) { nwords := rdr.Words() for nwords > 0 { nwords-- - wr.PutNextWord(rdr.NextWord()) + if mode == transferInvert { + wr.PutNextWord(^rdr.NextWord()) + } else { + wr.PutNextWord(rdr.NextWord()) + } } nbytes := rdr.TrailingBytes() for nbytes > 0 { nbytes-- bt, validBits := rdr.NextTrailingByte() + if mode == transferInvert { + bt = ^bt + } wr.PutNextTrailingByte(bt, validBits) } return @@ -417,14 +430,33 @@ func CopyBitmap(src []byte, srcOffset, length int, dst []byte, dstOffset int) { // - high 5 bits: old bits from last byte of dest buffer trailingBits := nbytes*8 - length trailMask := byte(uint(1)<<(8-trailingBits)) - 1 - - copy(dst, src[:nbytes-1]) - lastData := src[nbytes-1] + var lastData byte + if mode == transferInvert { + for i, b := range src[:nbytes-1] { + dst[i] = ^b + } + lastData = ^src[nbytes-1] + } else { + copy(dst, src[:nbytes-1]) + lastData = src[nbytes-1] + } dst[nbytes-1] &= ^trailMask dst[nbytes-1] |= lastData & trailMask } +// CopyBitmap copies the bitmap indicated by src, starting at bit offset srcOffset, +// and copying length bits into dst, starting at bit offset dstOffset. +func CopyBitmap(src []byte, srcOffset, length int, dst []byte, dstOffset int) { + transferBitmap(transferCopy, src, srcOffset, length, dst, dstOffset) +} + +// InvertBitmap copies a bit range of a bitmap, inverting it as it copies +// over into the destination. +func InvertBitmap(src []byte, srcOffset, length int, dst []byte, dstOffset int) { + transferBitmap(transferInvert, src, srcOffset, length, dst, dstOffset) +} + type bitOp struct { opWord func(uint64, uint64) uint64 opByte func(byte, byte) byte @@ -440,6 +472,14 @@ var ( opWord: func(l, r uint64) uint64 { return l | r }, opByte: func(l, r byte) byte { return l | r }, } + bitAndNotOp = bitOp{ + opWord: func(l, r uint64) uint64 { return l &^ r }, + opByte: func(l, r byte) byte { return l &^ r }, + } + bitXorOp = bitOp{ + opWord: func(l, r uint64) uint64 { return l ^ r }, + opByte: func(l, r byte) byte { return l ^ r }, + } ) func alignedBitmapOp(op bitOp, left, right []byte, lOffset, rOffset int64, out []byte, outOffset int64, length int64) { @@ -532,6 +572,22 @@ func BitmapOrAlloc(mem memory.Allocator, left, right []byte, lOffset, rOffset in return BitmapOpAlloc(mem, bitOrOp, left, right, lOffset, rOffset, length, outOffset) } +func BitmapAndNot(left, right []byte, lOffset, rOffset int64, out []byte, outOffset int64, length int64) { + BitmapOp(bitAndNotOp, left, right, lOffset, rOffset, out, outOffset, length) +} + +func BitmapAndNotAlloc(mem memory.Allocator, left, right []byte, lOffset, rOffset int64, length, outOffset int64) *memory.Buffer { + return BitmapOpAlloc(mem, bitAndNotOp, left, right, lOffset, rOffset, length, outOffset) +} + +func BitmapXor(left, right []byte, lOffset, rOffset int64, out []byte, outOffset int64, length int64) { + BitmapOp(bitXorOp, left, right, lOffset, rOffset, out, outOffset, length) +} + +func BitmapXorAlloc(mem memory.Allocator, left, right []byte, lOffset, rOffset int64, length, outOffset int64) *memory.Buffer { + return BitmapOpAlloc(mem, bitXorOp, left, right, lOffset, rOffset, length, outOffset) +} + func BitmapEquals(left, right []byte, lOffset, rOffset int64, length int64) bool { if lOffset%8 == 0 && rOffset%8 == 0 { // byte aligned, fast path, can use bytes.Equal (memcmp) @@ -584,3 +640,108 @@ type OptionalBitIndexer struct { func (b *OptionalBitIndexer) GetBit(i int) bool { return b.Bitmap == nil || BitIsSet(b.Bitmap, b.Offset+i) } + +type Bitmap struct { + Data []byte + Offset, Len int64 +} + +func bitLength(bitmaps []Bitmap) (int64, error) { + for _, b := range bitmaps[1:] { + if b.Len != bitmaps[0].Len { + return -1, errors.New("bitmaps must be same length") + } + } + return bitmaps[0].Len, nil +} + +func runVisitWordsAndWriteLoop(bitLen int64, rdrs []*BitmapWordReader, wrs []*BitmapWordWriter, visitor func(in, out []uint64)) { + const bitWidth int64 = int64(uint64SizeBits) + + visited := make([]uint64, len(rdrs)) + output := make([]uint64, len(wrs)) + + // every reader will have same number of words, since they are same + // length'ed. This will be inefficient in some cases. When there's + // offsets beyond the Word boundary, every word would have to be + // created from 2 adjoining words + nwords := int64(rdrs[0].Words()) + bitLen -= nwords * bitWidth + for nwords > 0 { + nwords-- + for i := range visited { + visited[i] = rdrs[i].NextWord() + } + visitor(visited, output) + for i := range output { + wrs[i].PutNextWord(output[i]) + } + } + + // every reader will have the same number of trailing bytes, because + // we already confirmed they have the same length. Because + // offsets beyond the Word boundary can cause adjoining words, the + // tailing portion could be more than one word remaining full/partial + // words to write. + if bitLen == 0 { + return + } + + // convert the word visitor to a bytevisitor + byteVisitor := func(in, out []byte) { + for i, w := range in { + visited[i] = uint64(w) + } + visitor(visited, output) + for i, w := range output { + out[i] = byte(w) + } + } + + visitedBytes := make([]byte, len(rdrs)) + outputBytes := make([]byte, len(wrs)) + nbytes := rdrs[0].trailingBytes + for nbytes > 0 { + nbytes-- + memory.Set(visitedBytes, 0) + memory.Set(outputBytes, 0) + + var validBits int + for i := range rdrs { + visitedBytes[i], validBits = rdrs[i].NextTrailingByte() + } + byteVisitor(visitedBytes, outputBytes) + for i, w := range outputBytes { + wrs[i].PutNextTrailingByte(w, validBits) + } + } +} + +// VisitWordsAndWrite visits words of bits from each input bitmap and +// collects outputs to a slice of output Bitmaps. +// +// All bitmaps must have identical lengths. The first bit in a visited +// bitmap may be offset within the first visited word, but words will +// otherwise contain densely packed bits loaded from the bitmap. That +// offset within the first word is returned. +// +// NOTE: this function is efficient on 3+ sufficiently large bitmaps. +// It also has a large prolog/epilog overhead and should be used +// carefully in other cases. For 2 or fewer bitmaps, and/or smaller +// bitmaps, try BitmapReader and or other utilities. +func VisitWordsAndWrite(args []Bitmap, out []Bitmap, visitor func(in, out []uint64)) error { + bitLen, err := bitLength(args) + if err != nil { + return err + } + + rdrs, wrs := make([]*BitmapWordReader, len(args)), make([]*BitmapWordWriter, len(out)) + for i, in := range args { + rdrs[i] = NewBitmapWordReader(in.Data, int(in.Offset), int(in.Len)) + } + for i, o := range out { + wrs[i] = NewBitmapWordWriter(o.Data, int(o.Offset), int(o.Len)) + } + runVisitWordsAndWriteLoop(bitLen, rdrs, wrs, visitor) + return nil +} diff --git a/go/arrow/compute/arithmetic.go b/go/arrow/compute/arithmetic.go new file mode 100644 index 00000000000..49c3c24160e --- /dev/null +++ b/go/arrow/compute/arithmetic.go @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute + +import ( + "context" + "fmt" + + "github.com/apache/arrow/go/v10/arrow" + "github.com/apache/arrow/go/v10/arrow/compute/internal/exec" + "github.com/apache/arrow/go/v10/arrow/compute/internal/kernels" +) + +type arithmeticFunction struct { + ScalarFunction + + promote decimalPromotion +} + +func (fn *arithmeticFunction) checkDecimals(vals ...arrow.DataType) error { + if !hasDecimal(vals...) { + return nil + } + + if len(vals) != 2 { + return nil + } + + if fn.promote == decPromoteNone { + return fmt.Errorf("%w: invalid decimal function: %s", arrow.ErrInvalid, fn.name) + } + + return castBinaryDecimalArgs(fn.promote, vals...) +} + +func (fn *arithmeticFunction) DispatchBest(vals ...arrow.DataType) (exec.Kernel, error) { + if err := fn.checkArity(len(vals)); err != nil { + return nil, err + } + + if err := fn.checkDecimals(vals...); err != nil { + return nil, err + } + + if kn, err := fn.DispatchExact(vals...); err == nil { + return kn, nil + } + + ensureDictionaryDecoded(vals...) + + // only promote types for binary funcs + if len(vals) == 2 { + replaceNullWithOtherType(vals...) + if unit, istime := commonTemporalResolution(vals...); istime { + replaceTemporalTypes(unit, vals...) + } else { + if dt := commonNumeric(vals...); dt != nil { + replaceTypes(dt, vals...) + } + } + } + + return fn.DispatchExact(vals...) +} + +var ( + addDoc FunctionDoc +) + +func RegisterScalarArithmetic(reg FunctionRegistry) { + addFn := &arithmeticFunction{*NewScalarFunction("add_unchecked", Binary(), addDoc), decPromoteAdd} + for _, k := range kernels.GetArithmeticKernels(kernels.OpAdd) { + if err := addFn.AddKernel(k); err != nil { + panic(err) + } + } + + reg.AddFunction(addFn, false) + + addCheckedFn := &arithmeticFunction{*NewScalarFunction("add", Binary(), addDoc), decPromoteAdd} + for _, k := range kernels.GetArithmeticKernels(kernels.OpAddChecked) { + if err := addCheckedFn.AddKernel(k); err != nil { + panic(err) + } + } + + reg.AddFunction(addCheckedFn, false) + + subFn := &arithmeticFunction{*NewScalarFunction("sub_unchecked", Binary(), addDoc), decPromoteAdd} + for _, k := range kernels.GetArithmeticKernels(kernels.OpSub) { + if err := subFn.AddKernel(k); err != nil { + panic(err) + } + } + + reg.AddFunction(subFn, false) + + subCheckedFn := &arithmeticFunction{*NewScalarFunction("sub", Binary(), addDoc), decPromoteAdd} + for _, k := range kernels.GetArithmeticKernels(kernels.OpSubChecked) { + if err := subCheckedFn.AddKernel(k); err != nil { + panic(err) + } + } + + reg.AddFunction(subCheckedFn, false) +} + +// Add performs an addition between the passed in arguments (scalar or array) +// and returns the result. If one argument is a scalar and the other is an +// array, the scalar value is added to each value of the array. +// +// ArithmeticOptions specifies whether or not to check for overflows, +// performance is faster if not explicitly checking for overflows but +// will error on an overflow if CheckOverflow is true. +func Add(ctx context.Context, opts ArithmeticOptions, left, right Datum) (Datum, error) { + fn := "add" + if opts.NoCheckOverflow { + fn = "add_unchecked" + } + return CallFunction(ctx, fn, nil, left, right) +} + +// Sub performs a subtraction between the passed in arguments (scalar or array) +// and returns the result. If one argument is a scalar and the other is an +// array, the scalar value is subtracted from each value of the array. +// +// ArithmeticOptions specifies whether or not to check for overflows, +// performance is faster if not explicitly checking for overflows but +// will error on an overflow if CheckOverflow is true. +func Subtract(ctx context.Context, opts ArithmeticOptions, left, right Datum) (Datum, error) { + fn := "sub" + if opts.NoCheckOverflow { + fn = "sub_unchecked" + } + return CallFunction(ctx, fn, nil, left, right) +} diff --git a/go/arrow/compute/arithmetic_test.go b/go/arrow/compute/arithmetic_test.go new file mode 100644 index 00000000000..2da7a62fe86 --- /dev/null +++ b/go/arrow/compute/arithmetic_test.go @@ -0,0 +1,502 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute_test + +import ( + "context" + "fmt" + "math" + "strings" + "testing" + + "github.com/apache/arrow/go/v10/arrow" + "github.com/apache/arrow/go/v10/arrow/array" + "github.com/apache/arrow/go/v10/arrow/compute" + "github.com/apache/arrow/go/v10/arrow/compute/internal/exec" + "github.com/apache/arrow/go/v10/arrow/internal/testing/gen" + "github.com/apache/arrow/go/v10/arrow/memory" + "github.com/apache/arrow/go/v10/arrow/scalar" + "github.com/klauspost/cpuid/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +var ( + CpuCacheSizes = [...]int{ // defaults + 32 * 1024, // level 1: 32K + 256 * 1024, // level 2: 256K + 3072 * 1024, // level 3: 3M + } +) + +func init() { + if cpuid.CPU.Cache.L1D != -1 { + CpuCacheSizes[0] = cpuid.CPU.Cache.L1D + } + if cpuid.CPU.Cache.L2 != -1 { + CpuCacheSizes[1] = cpuid.CPU.Cache.L2 + } + if cpuid.CPU.Cache.L3 != -1 { + CpuCacheSizes[2] = cpuid.CPU.Cache.L3 + } +} + +type binaryArithmeticFunc = func(context.Context, compute.ArithmeticOptions, compute.Datum, compute.Datum) (compute.Datum, error) + +type binaryFunc = func(left, right compute.Datum) (compute.Datum, error) + +func assertScalarEquals(t *testing.T, expected, actual scalar.Scalar) { + assert.Truef(t, scalar.Equals(expected, actual), "expected: %s\ngot: %s", expected, actual) +} + +func assertBinop(t *testing.T, fn binaryFunc, left, right, expected arrow.Array) { + actual, err := fn(&compute.ArrayDatum{Value: left.Data()}, &compute.ArrayDatum{Value: right.Data()}) + require.NoError(t, err) + defer actual.Release() + assertDatumsEqual(t, &compute.ArrayDatum{Value: expected.Data()}, actual) + + // also check (Scalar, Scalar) operations + for i := 0; i < expected.Len(); i++ { + s, err := scalar.GetScalar(expected, i) + require.NoError(t, err) + lhs, _ := scalar.GetScalar(left, i) + rhs, _ := scalar.GetScalar(right, i) + + actual, err := fn(&compute.ScalarDatum{Value: lhs}, &compute.ScalarDatum{Value: rhs}) + assert.NoError(t, err) + assertScalarEquals(t, s, actual.(*compute.ScalarDatum).Value) + } +} + +func assertBinopErr(t *testing.T, fn binaryFunc, left, right arrow.Array, expectedMsg string) { + _, err := fn(&compute.ArrayDatum{left.Data()}, &compute.ArrayDatum{Value: right.Data()}) + assert.ErrorIs(t, err, arrow.ErrInvalid) + assert.ErrorContains(t, err, expectedMsg) +} + +type BinaryFuncTestSuite struct { + suite.Suite + + mem *memory.CheckedAllocator + ctx context.Context +} + +func (b *BinaryFuncTestSuite) SetupTest() { + b.mem = memory.NewCheckedAllocator(memory.DefaultAllocator) + b.ctx = compute.WithAllocator(context.TODO(), b.mem) +} + +func (b *BinaryFuncTestSuite) TearDownTest() { + b.mem.AssertSize(b.T(), 0) +} + +type Float16BinaryFuncTestSuite struct { + BinaryFuncTestSuite +} + +func (b *Float16BinaryFuncTestSuite) assertBinopErr(fn binaryFunc, lhs, rhs string) { + left, _, _ := array.FromJSON(b.mem, arrow.FixedWidthTypes.Float16, strings.NewReader(lhs), array.WithUseNumber()) + defer left.Release() + right, _, _ := array.FromJSON(b.mem, arrow.FixedWidthTypes.Float16, strings.NewReader(rhs), array.WithUseNumber()) + defer right.Release() + + _, err := fn(&compute.ArrayDatum{left.Data()}, &compute.ArrayDatum{right.Data()}) + b.ErrorIs(err, arrow.ErrNotImplemented) +} + +func (b *Float16BinaryFuncTestSuite) TestAdd() { + for _, overflow := range []bool{false, true} { + b.Run(fmt.Sprintf("no_overflow_check=%t", overflow), func() { + opts := compute.ArithmeticOptions{NoCheckOverflow: overflow} + b.assertBinopErr(func(left, right compute.Datum) (compute.Datum, error) { + return compute.Add(b.ctx, opts, left, right) + }, `[1.5]`, `[1.5]`) + }) + } +} + +func (b *Float16BinaryFuncTestSuite) TestSub() { + for _, overflow := range []bool{false, true} { + b.Run(fmt.Sprintf("no_overflow_check=%t", overflow), func() { + opts := compute.ArithmeticOptions{NoCheckOverflow: overflow} + b.assertBinopErr(func(left, right compute.Datum) (compute.Datum, error) { + return compute.Subtract(b.ctx, opts, left, right) + }, `[1.5]`, `[1.5]`) + }) + } +} + +type BinaryArithmeticSuite[T exec.NumericTypes] struct { + BinaryFuncTestSuite + + opts compute.ArithmeticOptions + min, max T +} + +func (BinaryArithmeticSuite[T]) DataType() arrow.DataType { + return exec.GetDataType[T]() +} + +func (b *BinaryArithmeticSuite[T]) SetupTest() { + b.BinaryFuncTestSuite.SetupTest() + b.opts.NoCheckOverflow = false +} + +func (b *BinaryArithmeticSuite[T]) makeNullScalar() scalar.Scalar { + return scalar.MakeNullScalar(b.DataType()) +} + +func (b *BinaryArithmeticSuite[T]) makeScalar(val T) scalar.Scalar { + return scalar.MakeScalar(val) +} + +func (b *BinaryArithmeticSuite[T]) assertBinopScalars(fn binaryArithmeticFunc, lhs, rhs T, expected T) { + left, right := b.makeScalar(lhs), b.makeScalar(rhs) + exp := b.makeScalar(expected) + + actual, err := fn(b.ctx, b.opts, &compute.ScalarDatum{Value: left}, &compute.ScalarDatum{Value: right}) + b.NoError(err) + sc := actual.(*compute.ScalarDatum).Value + + assertScalarEquals(b.T(), exp, sc) +} + +func (b *BinaryArithmeticSuite[T]) assertBinopScalarValArr(fn binaryArithmeticFunc, lhs T, rhs, expected string) { + left := b.makeScalar(lhs) + b.assertBinopScalarArr(fn, left, rhs, expected) +} + +func (b *BinaryArithmeticSuite[T]) assertBinopScalarArr(fn binaryArithmeticFunc, lhs scalar.Scalar, rhs, expected string) { + right, _, _ := array.FromJSON(b.mem, b.DataType(), strings.NewReader(rhs)) + defer right.Release() + exp, _, _ := array.FromJSON(b.mem, b.DataType(), strings.NewReader(expected)) + defer exp.Release() + + actual, err := fn(b.ctx, b.opts, &compute.ScalarDatum{Value: lhs}, &compute.ArrayDatum{Value: right.Data()}) + b.NoError(err) + defer actual.Release() + assertDatumsEqual(b.T(), &compute.ArrayDatum{Value: exp.Data()}, actual) +} + +func (b *BinaryArithmeticSuite[T]) assertBinopArrScalarVal(fn binaryArithmeticFunc, lhs string, rhs T, expected string) { + right := b.makeScalar(rhs) + b.assertBinopArrScalar(fn, lhs, right, expected) +} + +func (b *BinaryArithmeticSuite[T]) assertBinopArrScalar(fn binaryArithmeticFunc, lhs string, rhs scalar.Scalar, expected string) { + left, _, _ := array.FromJSON(b.mem, b.DataType(), strings.NewReader(lhs)) + defer left.Release() + exp, _, _ := array.FromJSON(b.mem, b.DataType(), strings.NewReader(expected)) + defer exp.Release() + + actual, err := fn(b.ctx, b.opts, &compute.ArrayDatum{Value: left.Data()}, &compute.ScalarDatum{Value: rhs}) + b.NoError(err) + defer actual.Release() + assertDatumsEqual(b.T(), &compute.ArrayDatum{Value: exp.Data()}, actual) +} + +func (b *BinaryArithmeticSuite[T]) assertBinop(fn binaryArithmeticFunc, lhs, rhs, expected string) { + left, _, _ := array.FromJSON(b.mem, b.DataType(), strings.NewReader(lhs)) + defer left.Release() + right, _, _ := array.FromJSON(b.mem, b.DataType(), strings.NewReader(rhs)) + defer right.Release() + exp, _, _ := array.FromJSON(b.mem, b.DataType(), strings.NewReader(expected)) + defer exp.Release() + + assertBinop(b.T(), func(left, right compute.Datum) (compute.Datum, error) { + return fn(b.ctx, b.opts, left, right) + }, left, right, exp) +} + +func (b *BinaryArithmeticSuite[T]) setOverflowCheck(value bool) { + b.opts.NoCheckOverflow = value +} + +func (b *BinaryArithmeticSuite[T]) assertBinopErr(fn binaryArithmeticFunc, lhs, rhs, expectedMsg string) { + left, _, _ := array.FromJSON(b.mem, b.DataType(), strings.NewReader(lhs), array.WithUseNumber()) + defer left.Release() + right, _, _ := array.FromJSON(b.mem, b.DataType(), strings.NewReader(rhs), array.WithUseNumber()) + defer right.Release() + + assertBinopErr(b.T(), func(left, right compute.Datum) (compute.Datum, error) { + return fn(b.ctx, b.opts, left, right) + }, left, right, expectedMsg) +} + +func (b *BinaryArithmeticSuite[T]) TestAdd() { + b.Run(b.DataType().String(), func() { + for _, overflow := range []bool{false, true} { + b.Run(fmt.Sprintf("no_overflow_check=%t", overflow), func() { + b.setOverflowCheck(overflow) + + b.assertBinop(compute.Add, `[]`, `[]`, `[]`) + b.assertBinop(compute.Add, `[3, 2, 6]`, `[1, 0, 2]`, `[4, 2, 8]`) + // nulls on one side + b.assertBinop(compute.Add, `[null, 1, null]`, `[3, 4, 5]`, `[null, 5, null]`) + b.assertBinop(compute.Add, `[3, 4, 5]`, `[null, 1, null]`, `[null, 5, null]`) + // nulls on both sides + b.assertBinop(compute.Add, `[null, 1, 2]`, `[3, 4, null]`, `[null, 5, null]`) + // all nulls + b.assertBinop(compute.Add, `[null]`, `[null]`, `[null]`) + + // scalar on the left + b.assertBinopScalarValArr(compute.Add, 3, `[1, 2]`, `[4, 5]`) + b.assertBinopScalarValArr(compute.Add, 3, `[null, 2]`, `[null, 5]`) + b.assertBinopScalarArr(compute.Add, b.makeNullScalar(), `[1, 2]`, `[null, null]`) + b.assertBinopScalarArr(compute.Add, b.makeNullScalar(), `[null, 2]`, `[null, null]`) + // scalar on the right + b.assertBinopArrScalarVal(compute.Add, `[1, 2]`, 3, `[4, 5]`) + b.assertBinopArrScalarVal(compute.Add, `[null, 2]`, 3, `[null, 5]`) + b.assertBinopArrScalar(compute.Add, `[1, 2]`, b.makeNullScalar(), `[null, null]`) + b.assertBinopArrScalar(compute.Add, `[null, 2]`, b.makeNullScalar(), `[null, null]`) + + if !arrow.IsFloating(b.DataType().ID()) && !overflow { + val := fmt.Sprintf("[%v]", b.max) + b.assertBinopErr(compute.Add, val, val, "overflow") + } + }) + } + }) +} + +func (b *BinaryArithmeticSuite[T]) TestSub() { + b.Run(b.DataType().String(), func() { + for _, overflow := range []bool{false, true} { + b.Run(fmt.Sprintf("no_overflow_check=%t", overflow), func() { + b.setOverflowCheck(overflow) + + b.assertBinop(compute.Subtract, `[]`, `[]`, `[]`) + b.assertBinop(compute.Subtract, `[3, 2, 6]`, `[1, 0, 2]`, `[2, 2, 4]`) + // nulls on one side + b.assertBinop(compute.Subtract, `[null, 4, null]`, `[2, 1, 0]`, `[null, 3, null]`) + b.assertBinop(compute.Subtract, `[3, 4, 5]`, `[null, 1, null]`, `[null, 3, null]`) + // nulls on both sides + b.assertBinop(compute.Subtract, `[null, 4, 3]`, `[2, 1, null]`, `[null, 3, null]`) + // all nulls + b.assertBinop(compute.Subtract, `[null]`, `[null]`, `[null]`) + + // scalar on the left + b.assertBinopScalarValArr(compute.Subtract, 3, `[1, 2]`, `[2, 1]`) + b.assertBinopScalarValArr(compute.Subtract, 3, `[null, 2]`, `[null, 1]`) + b.assertBinopScalarArr(compute.Subtract, b.makeNullScalar(), `[1, 2]`, `[null, null]`) + b.assertBinopScalarArr(compute.Subtract, b.makeNullScalar(), `[null, 2]`, `[null, null]`) + // scalar on the right + b.assertBinopArrScalarVal(compute.Subtract, `[4, 5]`, 3, `[1, 2]`) + b.assertBinopArrScalarVal(compute.Subtract, `[null, 5]`, 3, `[null, 2]`) + b.assertBinopArrScalar(compute.Subtract, `[1, 2]`, b.makeNullScalar(), `[null, null]`) + b.assertBinopArrScalar(compute.Subtract, `[null, 2]`, b.makeNullScalar(), `[null, null]`) + + if !arrow.IsFloating(b.DataType().ID()) && !overflow { + b.assertBinopErr(compute.Subtract, fmt.Sprintf("[%v]", b.min), fmt.Sprintf("[%v]", b.max), "overflow") + } + }) + } + }) +} + +func TestBinaryArithmetic(t *testing.T) { + suite.Run(t, &BinaryArithmeticSuite[int8]{min: math.MinInt8, max: math.MaxInt8}) + suite.Run(t, &BinaryArithmeticSuite[uint8]{min: 0, max: math.MaxUint8}) + suite.Run(t, &BinaryArithmeticSuite[int16]{min: math.MinInt16, max: math.MaxInt16}) + suite.Run(t, &BinaryArithmeticSuite[uint16]{min: 0, max: math.MaxUint16}) + suite.Run(t, &BinaryArithmeticSuite[int32]{min: math.MinInt32, max: math.MaxInt32}) + suite.Run(t, &BinaryArithmeticSuite[uint32]{min: 0, max: math.MaxUint32}) + suite.Run(t, &BinaryArithmeticSuite[int64]{min: math.MinInt64, max: math.MaxInt64}) + suite.Run(t, &BinaryArithmeticSuite[uint64]{min: 0, max: math.MaxUint64}) + suite.Run(t, &BinaryArithmeticSuite[float32]{min: -math.MaxFloat32, max: math.MaxFloat32}) + suite.Run(t, &BinaryArithmeticSuite[float64]{min: -math.MaxFloat64, max: math.MaxFloat64}) + suite.Run(t, new(Float16BinaryFuncTestSuite)) +} + +func TestBinaryArithmeticDispatchBest(t *testing.T) { + for _, name := range []string{"add", "sub"} { + for _, suffix := range []string{"", "_unchecked"} { + name += suffix + t.Run(name, func(t *testing.T) { + + tests := []struct { + left, right arrow.DataType + expected arrow.DataType + }{ + {arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int32}, + {arrow.PrimitiveTypes.Int32, arrow.Null, arrow.PrimitiveTypes.Int32}, + {arrow.Null, arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int32}, + {arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int8, arrow.PrimitiveTypes.Int32}, + {arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int16, arrow.PrimitiveTypes.Int32}, + {arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int32}, + {arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int64, arrow.PrimitiveTypes.Int64}, + {arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Uint8, arrow.PrimitiveTypes.Int32}, + {arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Uint16, arrow.PrimitiveTypes.Int32}, + {arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Uint32, arrow.PrimitiveTypes.Int64}, + {arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Uint64, arrow.PrimitiveTypes.Int64}, + {arrow.PrimitiveTypes.Uint8, arrow.PrimitiveTypes.Uint8, arrow.PrimitiveTypes.Uint8}, + {arrow.PrimitiveTypes.Uint8, arrow.PrimitiveTypes.Uint16, arrow.PrimitiveTypes.Uint16}, + {arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Float32, arrow.PrimitiveTypes.Float32}, + {arrow.PrimitiveTypes.Float32, arrow.PrimitiveTypes.Int64, arrow.PrimitiveTypes.Float32}, + {arrow.PrimitiveTypes.Float64, arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Float64}, + {&arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.PrimitiveTypes.Float64}, + arrow.PrimitiveTypes.Float64, arrow.PrimitiveTypes.Float64}, + {&arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int8, ValueType: arrow.PrimitiveTypes.Float64}, + arrow.PrimitiveTypes.Int16, arrow.PrimitiveTypes.Float64}, + } + + for _, tt := range tests { + CheckDispatchBest(t, name, []arrow.DataType{tt.left, tt.right}, []arrow.DataType{tt.expected, tt.expected}) + } + }) + } + } +} + +const seed = 0x94378165 + +type binaryOp = func(ctx context.Context, left, right compute.Datum) (compute.Datum, error) + +func Add(ctx context.Context, left, right compute.Datum) (compute.Datum, error) { + var opts compute.ArithmeticOptions + return compute.Add(ctx, opts, left, right) +} + +func Subtract(ctx context.Context, left, right compute.Datum) (compute.Datum, error) { + var opts compute.ArithmeticOptions + return compute.Subtract(ctx, opts, left, right) +} + +func AddUnchecked(ctx context.Context, left, right compute.Datum) (compute.Datum, error) { + opts := compute.ArithmeticOptions{NoCheckOverflow: true} + return compute.Add(ctx, opts, left, right) +} + +func SubtractUnchecked(ctx context.Context, left, right compute.Datum) (compute.Datum, error) { + opts := compute.ArithmeticOptions{NoCheckOverflow: true} + return compute.Subtract(ctx, opts, left, right) +} + +func arrayScalarKernel(b *testing.B, sz int, nullProp float64, op binaryOp, dt arrow.DataType) { + b.Run("array scalar", func(b *testing.B) { + var ( + mem = memory.NewCheckedAllocator(memory.DefaultAllocator) + arraySize = int64(sz / dt.(arrow.FixedWidthDataType).Bytes()) + min int64 = 6 + max = min + 15 + sc, _ = scalar.MakeScalarParam(6, dt) + rhs compute.Datum = &compute.ScalarDatum{Value: sc} + rng = gen.NewRandomArrayGenerator(seed, mem) + ) + + lhs := rng.Numeric(dt.ID(), arraySize, min, max, nullProp) + b.Cleanup(func() { + lhs.Release() + }) + + var ( + res compute.Datum + err error + ctx = context.Background() + left = &compute.ArrayDatum{Value: lhs.Data()} + ) + + b.SetBytes(arraySize) + b.ResetTimer() + for n := 0; n < b.N; n++ { + res, err = op(ctx, left, rhs) + b.StopTimer() + if err != nil { + b.Fatal(err) + } + res.Release() + b.StartTimer() + } + }) +} + +func arrayArrayKernel(b *testing.B, sz int, nullProp float64, op binaryOp, dt arrow.DataType) { + b.Run("array array", func(b *testing.B) { + var ( + mem = memory.NewCheckedAllocator(memory.DefaultAllocator) + arraySize = int64(sz / dt.(arrow.FixedWidthDataType).Bytes()) + rmin int64 = 1 + rmax = rmin + 6 // 7 + lmin = rmax + 1 // 8 + lmax = lmin + 6 // 14 + rng = gen.NewRandomArrayGenerator(seed, mem) + ) + + lhs := rng.Numeric(dt.ID(), arraySize, lmin, lmax, nullProp) + rhs := rng.Numeric(dt.ID(), arraySize, rmin, rmax, nullProp) + b.Cleanup(func() { + lhs.Release() + rhs.Release() + }) + var ( + res compute.Datum + err error + ctx = context.Background() + left = &compute.ArrayDatum{Value: lhs.Data()} + right = &compute.ArrayDatum{Value: rhs.Data()} + ) + + b.SetBytes(arraySize) + b.ResetTimer() + for n := 0; n < b.N; n++ { + res, err = op(ctx, left, right) + b.StopTimer() + if err != nil { + b.Fatal(err) + } + res.Release() + b.StartTimer() + } + }) +} + +func BenchmarkScalarArithmetic(b *testing.B) { + args := []struct { + sz int + nullProb float64 + }{ + {CpuCacheSizes[2], 0}, + {CpuCacheSizes[2], 0.5}, + {CpuCacheSizes[2], 1}, + } + + testfns := []struct { + name string + op binaryOp + }{ + {"Add", Add}, + {"AddUnchecked", AddUnchecked}, + {"Subtract", Subtract}, + {"SubtractUnchecked", SubtractUnchecked}, + } + + for _, dt := range numericTypes { + b.Run(dt.String(), func(b *testing.B) { + for _, benchArgs := range args { + b.Run(fmt.Sprintf("sz=%d/nullprob=%.2f", benchArgs.sz, benchArgs.nullProb), func(b *testing.B) { + for _, tfn := range testfns { + b.Run(tfn.name, func(b *testing.B) { + arrayArrayKernel(b, benchArgs.sz, benchArgs.nullProb, tfn.op, dt) + arrayScalarKernel(b, benchArgs.sz, benchArgs.nullProb, tfn.op, dt) + }) + } + }) + } + }) + } +} diff --git a/go/arrow/compute/cast_test.go b/go/arrow/compute/cast_test.go index c8f07e23aef..cb5c4f8a758 100644 --- a/go/arrow/compute/cast_test.go +++ b/go/arrow/compute/cast_test.go @@ -34,7 +34,6 @@ import ( "github.com/apache/arrow/go/v10/arrow/internal/testing/types" "github.com/apache/arrow/go/v10/arrow/memory" "github.com/apache/arrow/go/v10/arrow/scalar" - "github.com/klauspost/cpuid/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -2732,26 +2731,6 @@ func TestCasts(t *testing.T) { const rngseed = 0x94378165 -var ( - CpuCacheSizes = [...]int{ // defaults - 32 * 1024, // level 1: 32K - 256 * 1024, // level 2: 256K - 3072 * 1024, // level 3: 3M - } -) - -func init() { - if cpuid.CPU.Cache.L1D != -1 { - CpuCacheSizes[0] = cpuid.CPU.Cache.L1D - } - if cpuid.CPU.Cache.L2 != -1 { - CpuCacheSizes[1] = cpuid.CPU.Cache.L2 - } - if cpuid.CPU.Cache.L3 != -1 { - CpuCacheSizes[2] = cpuid.CPU.Cache.L3 - } -} - func benchmarkNumericCast(b *testing.B, fromType, toType arrow.DataType, opts compute.CastOptions, size, min, max int64, nullprob float64) { rng := gen.NewRandomArrayGenerator(rngseed, memory.DefaultAllocator) arr := rng.Numeric(fromType.ID(), size, min, max, nullprob) diff --git a/go/arrow/compute/exec.go b/go/arrow/compute/exec.go index 3709424b9e4..b7f4962806c 100644 --- a/go/arrow/compute/exec.go +++ b/go/arrow/compute/exec.go @@ -99,6 +99,17 @@ func execInternal(ctx context.Context, fn Function, opts FunctionOptions, passed return } + // cast arguments if necessary + for i, arg := range args { + if !arrow.TypeEqual(inTypes[i], arg.(ArrayLikeDatum).Type()) { + args[i], err = CastDatum(ctx, arg, SafeCastOptions(inTypes[i])) + if err != nil { + return nil, err + } + defer args[i].Release() + } + } + kctx := &exec.KernelCtx{Ctx: ctx, Kernel: k} init := k.GetInitFn() kinitArgs := exec.KernelInitArgs{Kernel: k, Inputs: inTypes, Options: opts} diff --git a/go/arrow/compute/executor.go b/go/arrow/compute/executor.go index 8098f2f8edd..f51c59deaf0 100644 --- a/go/arrow/compute/executor.go +++ b/go/arrow/compute/executor.go @@ -242,7 +242,7 @@ func propagateNulls(ctx *exec.KernelCtx, batch *exec.ExecSpan, out *exec.ArraySp } var ( - arrsWithNulls = make([]*exec.ArraySpan, 0) + arrsWithNulls = make([]*exec.ArraySpan, 0, len(batch.Values)) isAllNull bool prealloc bool = out.Buffers[0].Buf != nil ) @@ -596,6 +596,7 @@ func (s *scalarExecutor) executeSpans(data chan<- Datum) (err error) { resultOffset = nextOffset } if err != nil { + prealloc.Release() return } diff --git a/go/arrow/compute/expression.go b/go/arrow/compute/expression.go index 644de5cf5c9..aa6e3661afa 100644 --- a/go/arrow/compute/expression.go +++ b/go/arrow/compute/expression.go @@ -485,7 +485,7 @@ const ( ) type ArithmeticOptions struct { - CheckOverflow bool `compute:"check_overflow"` + NoCheckOverflow bool `compute:"check_overflow"` } func (ArithmeticOptions) TypeName() string { return "ArithmeticOptions" } diff --git a/go/arrow/compute/functions_test.go b/go/arrow/compute/functions_test.go index 78dbd8be5e4..1f167f0232c 100644 --- a/go/arrow/compute/functions_test.go +++ b/go/arrow/compute/functions_test.go @@ -19,8 +19,10 @@ package compute_test import ( "testing" + "github.com/apache/arrow/go/v10/arrow" "github.com/apache/arrow/go/v10/arrow/compute" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestArityBasics(t *testing.T) { @@ -44,3 +46,22 @@ func TestArityBasics(t *testing.T) { assert.Equal(t, 2, varargs.NArgs) assert.True(t, varargs.IsVarArgs) } + +func CheckDispatchBest(t *testing.T, funcName string, originalTypes, expected []arrow.DataType) { + fn, exists := compute.GetFunctionRegistry().GetFunction(funcName) + require.True(t, exists) + + vals := make([]arrow.DataType, len(originalTypes)) + copy(vals, originalTypes) + + actualKernel, err := fn.DispatchBest(vals...) + require.NoError(t, err) + expKernel, err := fn.DispatchExact(expected...) + require.NoError(t, err) + + assert.Same(t, expKernel, actualKernel) + assert.Equal(t, len(expected), len(vals)) + for i, v := range vals { + assert.True(t, arrow.TypeEqual(v, expected[i]), v.String(), expected[i].String()) + } +} diff --git a/go/arrow/compute/internal/exec/span.go b/go/arrow/compute/internal/exec/span.go index ca6caf436b9..1e8a719d347 100644 --- a/go/arrow/compute/internal/exec/span.go +++ b/go/arrow/compute/internal/exec/span.go @@ -86,6 +86,21 @@ type ArraySpan struct { Children []ArraySpan } +// if an error is encountered, call Release on a preallocated span +// to ensure it releases any self-allocated buffers, it will +// not call release on buffers it doesn't own (SelfAlloc != true) +func (a *ArraySpan) Release() { + for _, c := range a.Children { + c.Release() + } + + for _, b := range a.Buffers { + if b.SelfAlloc { + b.Owner.Release() + } + } +} + func (a *ArraySpan) MayHaveNulls() bool { return atomic.LoadInt64(&a.Nulls) != 0 && a.Buffers[0].Buf != nil } @@ -114,7 +129,7 @@ func (a *ArraySpan) NumBuffers() int { return getNumBuffers(a.Type) } // MakeData generates an arrow.ArrayData object for this ArraySpan, // properly updating the buffer ref count if necessary. func (a *ArraySpan) MakeData() arrow.ArrayData { - bufs := make([]*memory.Buffer, a.NumBuffers()) + var bufs [3]*memory.Buffer for i := range bufs { b := a.GetBuffer(i) bufs[i] = b @@ -155,7 +170,7 @@ func (a *ArraySpan) MakeData() arrow.ArrayData { } if dt.ID() == arrow.DICTIONARY { - result := array.NewData(a.Type, length, bufs, nil, nulls, off) + result := array.NewData(a.Type, length, bufs[:a.NumBuffers()], nil, nulls, off) dict := a.Dictionary().MakeData() defer dict.Release() result.SetDictionary(dict) @@ -173,7 +188,7 @@ func (a *ArraySpan) MakeData() arrow.ArrayData { children[i] = d } } - return array.NewData(a.Type, length, bufs, children, nulls, off) + return array.NewData(a.Type, length, bufs[:a.NumBuffers()], children, nulls, off) } // MakeArray is a convenience function for calling array.MakeFromData(a.MakeData()) @@ -186,14 +201,24 @@ func (a *ArraySpan) MakeArray() arrow.Array { // SetSlice updates the offset and length of this ArraySpan to refer to // a specific slice of the underlying buffers. func (a *ArraySpan) SetSlice(off, length int64) { - a.Offset, a.Len = off, length + if off == a.Offset && length == a.Len { + // don't modify the nulls if the slice is the entire span + return + } + if a.Type.ID() != arrow.NULL { if a.Nulls != 0 { - a.Nulls = array.UnknownNullCount + if a.Nulls == a.Len { + a.Nulls = length + } else { + a.Nulls = array.UnknownNullCount + } } } else { - a.Nulls = a.Len + a.Nulls = length } + + a.Offset, a.Len = off, length } // GetBuffer returns the buffer for the requested index. If this buffer diff --git a/go/arrow/compute/internal/exec/utils.go b/go/arrow/compute/internal/exec/utils.go index 876e3f38ece..57fe3183c6e 100644 --- a/go/arrow/compute/internal/exec/utils.go +++ b/go/arrow/compute/internal/exec/utils.go @@ -135,6 +135,13 @@ func Min[T constraints.Ordered](a, b T) T { return b } +func Max[T constraints.Ordered](a, b T) T { + if a > b { + return a + } + return b +} + // OptionsInit should be used in the case where a KernelState is simply // represented with a specific type by value (instead of pointer). // This will initialize the KernelState as a value-copied instance of @@ -165,13 +172,26 @@ var typMap = map[reflect.Type]arrow.DataType{ reflect.TypeOf(arrow.Date32(0)): arrow.FixedWidthTypes.Date32, reflect.TypeOf(arrow.Date64(0)): arrow.FixedWidthTypes.Date64, reflect.TypeOf(true): arrow.FixedWidthTypes.Boolean, + reflect.TypeOf(float16.Num{}): arrow.FixedWidthTypes.Float16, } -func GetDataType[T NumericTypes | bool | string]() arrow.DataType { +// GetDataType returns the appropriate arrow.DataType for the given type T +// only for non-parametric types. This uses a map and reflection internally +// so don't call this in a tight loop, instead call this once and then use +// a closure with the result. +func GetDataType[T NumericTypes | bool | string | float16.Num]() arrow.DataType { var z T return typMap[reflect.TypeOf(z)] } +// GetType returns the appropriate arrow.Type type T, only for non-parameteric +// types. This uses a map and reflection internally so don't call this in +// a tight loop, instead call it once and then use a closure with the result. +func GetType[T NumericTypes | bool | string]() arrow.Type { + var z T + return typMap[reflect.TypeOf(z)].ID() +} + type arrayBuilder[T NumericTypes] interface { array.Builder Append(T) diff --git a/go/arrow/compute/internal/kernels/Makefile b/go/arrow/compute/internal/kernels/Makefile index 752c38d412d..96238cc9a12 100644 --- a/go/arrow/compute/internal/kernels/Makefile +++ b/go/arrow/compute/internal/kernels/Makefile @@ -36,7 +36,8 @@ ALL_SOURCES := $(shell find . -path ./_lib -prune -o -name '*.go' -name '*.s' -n .PHONEY: assembly INTEL_SOURCES := \ - cast_numeric_avx2_amd64.s cast_numeric_sse4_amd64.s constant_factor_avx2_amd64.s constant_factor_sse4_amd64.s + cast_numeric_avx2_amd64.s cast_numeric_sse4_amd64.s constant_factor_avx2_amd64.s \ + constant_factor_sse4_amd64.s base_arithmetic_avx2_amd64.s base_arithmetic_sse4_amd64.s # # ARROW-15336: DO NOT add the assembly target for Arm64 (ARM_SOURCES) until c2goasm added the Arm64 support. @@ -55,6 +56,15 @@ _lib/cast_numeric_sse4_amd64.s: _lib/cast_numeric.cc _lib/cast_numeric_neon.s: _lib/cast_numeric.cc $(CXX) -std=c++17 -S $(C_FLAGS_NEON) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@ +_lib/base_arithmetic_avx2_amd64.s: _lib/base_arithmetic.cc + $(CXX) -std=c++17 -S $(C_FLAGS) $(ASM_FLAGS_AVX2) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@ + +_lib/base_arithmetic_sse4_amd64.s: _lib/base_arithmetic.cc + $(CXX) -std=c++17 -S $(C_FLAGS) $(ASM_FLAGS_SSE4) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@ + +_lib/base_arithmetic_neon.s: _lib/base_arithmetic.cc + $(CXX) -std=c++17 -S $(C_FLAGS_NEON) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@ + _lib/constant_factor_avx2_amd64.s: _lib/constant_factor.c $(CC) -S $(C_FLAGS) $(ASM_FLAGS_AVX2) $^ -o $@ ; $(PERL_FIXUP_ROTATE) $@ @@ -76,6 +86,12 @@ constant_factor_avx2_amd64.s: _lib/constant_factor_avx2_amd64.s constant_factor_sse4_amd64.s: _lib/constant_factor_sse4_amd64.s $(C2GOASM) -a -f $^ $@ +base_arithmetic_avx2_amd64.s: _lib/base_arithmetic_avx2_amd64.s + $(C2GOASM) -a -f $^ $@ + +base_arithmetic_sse4_amd64.s: _lib/base_arithmetic_sse4_amd64.s + $(C2GOASM) -a -f $^ $@ + clean: rm -f $(INTEL_SOURCES) rm -f $(addprefix _lib/,$(INTEL_SOURCES)) diff --git a/go/arrow/compute/internal/kernels/_lib/base_arithmetic.cc b/go/arrow/compute/internal/kernels/_lib/base_arithmetic.cc new file mode 100644 index 00000000000..dc2234bfb35 --- /dev/null +++ b/go/arrow/compute/internal/kernels/_lib/base_arithmetic.cc @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include "types.h" +#include "vendored/safe-math.h" + +// Corresponds to equivalent ArithmeticOp enum in base_arithmetic.go +// for passing across which operation to perform. This allows simpler +// implementation at the cost of having to pass the extra int8 and +// perform a switch. +// +// In cases of small arrays, this is completely negligible. In cases +// of large arrays, the time saved by using SIMD here is significantly +// worth the cost. +enum class optype : int8_t { + ADD, + SUB, + + // this impl doesn't actually perform any overflow checks as we need + // to only run overflow checks on non-null entries + ADD_CHECKED, + SUB_CHECKED, +}; + +struct Add { + template + static constexpr T Call(Arg0 left, Arg1 right) { + if constexpr (is_arithmetic_v) + return left + right; + } +}; + +struct Sub { + template + static constexpr T Call(Arg0 left, Arg1 right) { + if constexpr (is_arithmetic_v) + return left - right; + } +}; + +struct AddChecked { + template + static constexpr T Call(Arg0 left, Arg1 right) { + static_assert(is_same::value && is_same::value, ""); + if constexpr(is_arithmetic_v) { + return left + right; + } + } +}; + + +struct SubChecked { + template + static constexpr T Call(Arg0 left, Arg1 right) { + static_assert(is_same::value && is_same::value, ""); + if constexpr(is_arithmetic_v) { + return left - right; + } + } +}; + +template +struct arithmetic_op_arr_arr_impl { + static inline void exec(const void* in_left, const void* in_right, void* out, const int len) { + const T* left = reinterpret_cast(in_left); + const T* right = reinterpret_cast(in_right); + T* output = reinterpret_cast(out); + + for (int i = 0; i < len; ++i) { + output[i] = Op::template Call(left[i], right[i]); + } + } +}; + +template +struct arithmetic_op_arr_scalar_impl { + static inline void exec(const void* in_left, const void* scalar_right, void* out, const int len) { + const T* left = reinterpret_cast(in_left); + const T right = *reinterpret_cast(scalar_right); + T* output = reinterpret_cast(out); + + for (int i = 0; i < len; ++i) { + output[i] = Op::template Call(left[i], right); + } + } +}; + +template +struct arithmetic_op_scalar_arr_impl { + static inline void exec(const void* scalar_left, const void* in_right, void* out, const int len) { + const T left = *reinterpret_cast(scalar_left); + const T* right = reinterpret_cast(in_right); + T* output = reinterpret_cast(out); + + for (int i = 0; i < len; ++i) { + output[i] = Op::template Call(left, right[i]); + } + } +}; + + +template typename Impl> +static inline void arithmetic_op(const int type, const void* in_left, const void* in_right, void* output, const int len) { + const auto intype = static_cast(type); + + switch (intype) { + case arrtype::UINT8: + return Impl::exec(in_left, in_right, output, len); + case arrtype::INT8: + return Impl::exec(in_left, in_right, output, len); + case arrtype::UINT16: + return Impl::exec(in_left, in_right, output, len); + case arrtype::INT16: + return Impl::exec(in_left, in_right, output, len); + case arrtype::UINT32: + return Impl::exec(in_left, in_right, output, len); + case arrtype::INT32: + return Impl::exec(in_left, in_right, output, len); + case arrtype::UINT64: + return Impl::exec(in_left, in_right, output, len); + case arrtype::INT64: + return Impl::exec(in_left, in_right, output, len); + case arrtype::FLOAT32: + return Impl::exec(in_left, in_right, output, len); + case arrtype::FLOAT64: + return Impl::exec(in_left, in_right, output, len); + default: + break; + } +} + +template