diff --git a/LICENSE.txt b/LICENSE.txt index 843cf4f6a5e9..a82c22aeceaa 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1990,12 +1990,14 @@ for PyArrow. Ibis is released under the Apache License, Version 2.0. This project includes code from the autobrew project. -* r/tools/autobrew and dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb - are based on code from the autobrew project. +The following files are based on code from the autobrew project: +* r/tools/autobrew +* dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +* dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb Copyright (c) 2019, Jeroen Ooms License: MIT -Homepage: https://github.com/jeroen/autobrew +Homepage: https://github.com/autobrew/ -------------------------------------------------------------------------------- diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb new file mode 100644 index 000000000000..79d1da2c212d --- /dev/null +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow-static.rb @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Cloned from https://github.com/autobrew/homebrew-core/blob/high-sierra/Formula/apache-arrow.rb +# Upstream any relevant changes (dependencies, build args ...) + +class ApacheArrowStatic < Formula + desc "Columnar in-memory analytics layer designed to accelerate big data" + homepage "https://arrow.apache.org/" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-8.0.0.9000/apache-arrow-8.0.0.9000.tar.gz" + # Uncomment and update to test on a release candidate + # mirror "https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-8.0.0-rc1/apache-arrow-8.0.0.tar.gz" + sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" + head "https://github.com/apache/arrow.git" + + bottle do + sha256 cellar: :any, arm64_big_sur: "ef89d21a110b89840cc6148add685d407e75bd633bc8f79625eb33d00e3694b4" + sha256 cellar: :any, big_sur: "6fcb9f55d44eb61d295a8020e039a0622bdc044b220cfffef855f3e3ab8057a1" + sha256 cellar: :any, catalina: "bf71b17766688077fb9b4e679f309742c16524015287dd3633758c679752c977" + end + + depends_on "boost" => :build + depends_on "cmake" => :build + depends_on "aws-sdk-cpp-static" + depends_on "brotli" + depends_on "lz4" + depends_on "snappy" + depends_on "thrift" + depends_on "zstd" + + conflicts_with "apache-arrow", because: "both install Arrow" + + def install + ENV.cxx11 + # https://github.com/Homebrew/homebrew-core/issues/94724 + # https://issues.apache.org/jira/browse/ARROW-15664 + ENV["HOMEBREW_OPTIMIZATION_LEVEL"] = "O2" + + args = %W[ + -DARROW_BUILD_SHARED=OFF + -DARROW_BUILD_UTILITIES=ON + -DARROW_COMPUTE=ON + -DARROW_CSV=ON + -DARROW_DATASET=ON + -DARROW_FILESYSTEM=ON + -DARROW_GCS=ON + -DARROW_JEMALLOC=ON + -DARROW_JSON=ON + -DARROW_MIMALLOC=ON + -DARROW_PARQUET=ON + -DARROW_S3=ON + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON + -DARROW_WITH_BROTLI=ON + -DARROW_WITH_BZ2=ON + -DARROW_WITH_LZ4=ON + -DARROW_WITH_SNAPPY=ON + -DARROW_WITH_ZLIB=ON + -DARROW_WITH_ZSTD=ON + -DLZ4_HOME=#{Formula["lz4"].prefix} + -DPARQUET_BUILD_EXECUTABLES=ON + -DTHRIFT_HOME=#{Formula["thrift"].prefix} + ] + + mkdir "build" + cd "build" do + system "cmake", "../cpp", *std_cmake_args, *args + system "make" + system "make", "install" + end + end + + test do + (testpath/"test.cpp").write <<~EOS + #include "arrow/api.h" + int main(void) { + arrow::int64(); + return 0; + } + EOS + system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", \ + "-larrow", "-larrow_bundled_dependencies", "-o", "test" + system "./test" + end +end diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index a4491a4cc24f..de0c65dae40e 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -16,6 +16,7 @@ # under the License. # https://github.com/autobrew/homebrew-core/blob/master/Formula/apache-arrow.rb + class ApacheArrow < Formula desc "Columnar in-memory analytics layer designed to accelerate big data" homepage "https://arrow.apache.org/" @@ -25,15 +26,15 @@ class ApacheArrow < Formula bottle do cellar :any - sha256 "a55211ba6f464681b7ca1b48defdad9cfbe1cf6fad8ff9ec875dc5a3c8f3c5ed" => :el_capitan_or_later + sha256 "9cd44700798638b5e3ee8774b3929f3fad815290d05572d1f39f01d6423eaad0" => :high_sierra root_url "https://autobrew.github.io/bottles" end # NOTE: if you add something here, be sure to add to PKG_LIBS in r/tools/autobrew depends_on "boost" => :build - depends_on "brotli" depends_on "cmake" => :build depends_on "aws-sdk-cpp" + depends_on "brotli" depends_on "lz4" depends_on "snappy" depends_on "thrift" @@ -49,14 +50,11 @@ def install -DARROW_DATASET=ON -DARROW_FILESYSTEM=ON -DARROW_GCS=ON - -DARROW_HDFS=OFF -DARROW_JEMALLOC=ON -DARROW_JSON=ON -DARROW_MIMALLOC=ON -DARROW_PARQUET=ON - -DARROW_PYTHON=OFF -DARROW_S3=ON - -DARROW_USE_GLOG=OFF -DARROW_VERBOSE_THIRDPARTY_BUILD=ON -DARROW_WITH_BROTLI=ON -DARROW_WITH_BZ2=ON @@ -64,9 +62,8 @@ def install -DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON - -DCMAKE_UNITY_BUILD=OFF - -DPARQUET_BUILD_EXECUTABLES=ON -DLZ4_HOME=#{Formula["lz4"].prefix} + -DPARQUET_BUILD_EXECUTABLES=ON -DTHRIFT_HOME=#{Formula["thrift"].prefix} ] @@ -86,7 +83,8 @@ def install return 0; } EOS - system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", "-larrow", "-o", "test" + system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", \ + "-larrow", "-larrow_bundled_dependencies", "-o", "test" system "./test" end end diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index c7f7c647b7b5..bbabad69a717 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -226,6 +226,21 @@ on: done {% endmacro %} +{%- macro pin_brew_formulae(is_fork = false) -%} +{#- This macro expects the cwd to be arrow/r -#} + # Copy all *brew formulae + cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow*.rb tools/ + + # Pin the git commit in the formula to match + cd tools + if [ "{{ is_fork }}" == "true" ]; then + sed -i.bak -E -e 's/apache\/arrow.git"$/{{ arrow.github_repo.split("/") | join("\/") }}.git", :revision => "'"{{ arrow.head }}"'"/' apache-arrow*.rb + else + sed -i.bak -E -e 's/arrow.git"$/arrow.git", :revision => "'"{{ arrow.head }}"'"/' apache-arrow*.rb + fi + rm -f apache-arrow*.rb.bak +{% endmacro %} + {%- macro github_change_r_pkg_version(is_fork, version = '\\2.\'\"$(date +%Y%m%d)\"\'' ) -%} - name: Modify version shell: bash @@ -236,16 +251,8 @@ on: DESCRIPTION head DESCRIPTION rm -f DESCRIPTION.bak - cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb tools/apache-arrow.rb - # Pin the git commit in the formula to match - cd tools - if [ "{{ is_fork }}" == "true" ]; then - sed -i.bak -E -e 's/apache\/arrow.git"$/{{ arrow.github_repo.split("/") | join("\/") }}.git", :revision => "'"{{ arrow.head }}"'"/' apache-arrow.rb - else - sed -i.bak -E -e 's/arrow.git"$/arrow.git", :revision => "'"{{ arrow.head }}"'"/' apache-arrow.rb - fi - rm -f apache-arrow.rb.bak + {{ pin_brew_formulae(is_fork) }} {% endmacro %} {%- macro github_test_r_src_pkg() -%} @@ -309,3 +316,11 @@ on: # Set envvar for later steps by appending to $GITHUB_ENV write(paste0("R_PROFILE_USER=", profile_path), file = Sys.getenv("GITHUB_ENV"), append = TRUE) {% endmacro %} + +{# Detect if we are using a fork or the upstream repo #} + {% set is_upstream_b = arrow.github_repo == 'apache/arrow' %} + {# use filter to cast to string and convert to lowercase to match yaml boolean #} + {% set is_fork = (not is_upstream_b)|lower %} + +{% set r_release = '4.2' %} +{% set r_oldrel = '4.1' %} diff --git a/dev/tasks/r/github.macos.autobrew.yml b/dev/tasks/r/github.macos.autobrew.yml index c6c56277de92..1633682a430b 100644 --- a/dev/tasks/r/github.macos.autobrew.yml +++ b/dev/tasks/r/github.macos.autobrew.yml @@ -16,44 +16,60 @@ # under the License. {% import 'macros.jinja' as macros with context %} +{% set is_fork = macros.is_fork %} {{ macros.github_header() }} jobs: autobrew: - name: "Autobrew" - runs-on: macOS-10.15 + name: "Autobrew {{ "${{ matrix.platform }}" }}" + runs-on: {{ "${{ matrix.platform }}" }} + strategy: + fail-fast: false + matrix: + platform: + - macos-11 + - macos-10.13 # self-hosted + r-version: + - {{ macros.r_release }} + - {{ macros.r_oldrel }} steps: {{ macros.github_checkout_arrow()|indent }} - - name: Configure autobrew script run: | + # minio is pre-installed on the self-hosted 10.13 runer + if [ {{ '${{ matrix.platform }}' }} != macos-10.13 ]; then + # install minio for tests + brew install minio + fi cd arrow/r - # Put the formula inside r/ so that it's included in the package build - cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb tools/apache-arrow.rb - # Pin the current commit in the formula to test so that we're not always pulling from master - sed -i.bak -E -e 's@https://github.com/apache/arrow.git"$@{{ arrow.remote }}.git", :revision => "{{ arrow.head }}"@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak - # Sometimes crossbow gives a remote URL with .git and sometimes not. Make sure there's only one - sed -i.bak -E -e 's@.git.git@.git@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak - # Get minio for S3 testing - brew install minio - - uses: r-lib/actions/setup-r@v1 - - name: Install dependencies - run: | - install.packages("remotes") - remotes::install_deps("arrow/r", dependencies = TRUE) - remotes::install_cran(c("rcmdcheck", "sys", "sessioninfo")) - shell: Rscript {0} - - name: Session info + {{ macros.pin_brew_formulae(is_fork)|indent }} + - uses: r-lib/actions/setup-r@v2 + if: matrix.platform != 'macos-10.13' + with: + r-version: {{ '${{ matrix.r-version }}' }} + use-public-rspm: true + - name: Setup R + if: matrix.platform == 'macos-10.13' run: | - options(width = 100) - pkgs <- installed.packages()[, "Package"] - sessioninfo::session_info(pkgs, include_base = TRUE) - shell: Rscript {0} + # rig is a system utility that allows for switching + # between pre-installed R version on the self-hosted runners + rig default {{ '${{ matrix.r-version }}' }} + rig system setup-user-lib + rig system add-pak + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + working-directory: 'arrow/r' + extra-packages: | + any::rcmdcheck + any::sys + any::readr - name: Check env: _R_CHECK_CRAN_INCOMING_: false + NOT_CRAN: true ARROW_USE_PKG_CONFIG: false + ARROW_R_DEV: true run: arrow/ci/scripts/r_test.sh arrow - name: Dump install logs run: cat arrow/r/check/arrow.Rcheck/00install.out diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 6e46b3ff439c..7ca8bd7a5f7f 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -21,12 +21,7 @@ # crossbow submit --param custom_version=8.5.3 r-binary-packages # if the param is unset defaults to the usual Ymd naming scheme {% set package_version = custom_version|default("\\2.\'\"$(date +%Y%m%d)\"\'") %} -# We need this as boolean and string -{% set is_upstream_b = arrow.github_repo == 'apache/arrow' %} -# use filter to cast to string and convert to lowercase to match yaml boolean -{% set is_fork = (not is_upstream_b)|lower %} -{% set is_upstream = is_upstream_b|lower %} - +{% set is_fork = macros.is_fork %} {{ macros.github_header() }} diff --git a/r/tools/autobrew b/r/tools/autobrew index ea46be2c0d19..55fe9576086f 100644 --- a/r/tools/autobrew +++ b/r/tools/autobrew @@ -19,8 +19,15 @@ export HOMEBREW_NO_ANALYTICS=1 export HOMEBREW_NO_AUTO_UPDATE=1 -# Official Homebrew no longer supports El-Capitan -UPSTREAM_ORG="autobrew" +if [[ ${OSTYPE:6} -ge 20 ]]; then + # We are on a modern enough macOS, we can use the real brew + UPSTREAM_ORG="homebrew" + PKG_BREW_NAME="$PKG_BREW_NAME-static" +else + # Official Homebrew no longer supports El-Capitan + # so we need to use the forked autobrew version of brew that supports old macOSes + UPSTREAM_ORG="autobrew" +fi if [ "$DISABLE_AUTOBREW" ]; then return 0; fi AUTOBREW=${TMPDIR-/tmp} @@ -36,6 +43,13 @@ curl -fsSL https://github.com/$UPSTREAM_ORG/brew/tarball/master | tar xz --strip export HOMEBREW_CACHE="$AUTOBREW" LOCAL_FORMULA="tools/${PKG_BREW_NAME}.rb" if [ -f "$LOCAL_FORMULA" ]; then + if [[ ${OSTYPE:6} -ge 20 ]]; then + $BREW tap + + # Tap https://github.com/autobrew/homebrew-cran so that we can get dependencies from there + $BREW tap autobrew/cran + fi + # Use the local brew formula and install --HEAD $BREW deps -n "$LOCAL_FORMULA" 2>/dev/null BREW_DEPS=$($BREW deps -n "$LOCAL_FORMULA" 2>/dev/null)