diff --git a/.ci_support/migrations/grpc_cpp129.yaml b/.ci_support/migrations/grpc_cpp129.yaml deleted file mode 100644 index acdcd29d8..000000000 --- a/.ci_support/migrations/grpc_cpp129.yaml +++ /dev/null @@ -1,7 +0,0 @@ -__migrator: - build_number: 1 - kind: version - migration_number: 1 -grpc_cpp: -- '1.29' -migrator_ts: 1589470327.4445853 diff --git a/.ci_support/migrations/re2202005.yaml b/.ci_support/migrations/re2202005.yaml deleted file mode 100644 index a3880c801..000000000 --- a/.ci_support/migrations/re2202005.yaml +++ /dev/null @@ -1,7 +0,0 @@ -__migrator: - build_number: 1 - kind: version - migration_number: 1 -re2: -- 2020.05.01 -migrator_ts: 1589781985 diff --git a/README.md b/README.md index 8442bdc6e..1cfd4ee49 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,7 @@ Current release info | --- | --- | --- | --- | | [![Conda Recipe](https://img.shields.io/badge/recipe-arrow--cpp-green.svg)](https://anaconda.org/conda-forge/arrow-cpp) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/arrow-cpp.svg)](https://anaconda.org/conda-forge/arrow-cpp) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/arrow-cpp.svg)](https://anaconda.org/conda-forge/arrow-cpp) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/arrow-cpp.svg)](https://anaconda.org/conda-forge/arrow-cpp) | | [![Conda Recipe](https://img.shields.io/badge/recipe-arrow--cpp--proc-green.svg)](https://anaconda.org/conda-forge/arrow-cpp-proc) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/arrow-cpp-proc.svg)](https://anaconda.org/conda-forge/arrow-cpp-proc) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/arrow-cpp-proc.svg)](https://anaconda.org/conda-forge/arrow-cpp-proc) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/arrow-cpp-proc.svg)](https://anaconda.org/conda-forge/arrow-cpp-proc) | +| [![Conda Recipe](https://img.shields.io/badge/recipe-pyarrow-green.svg)](https://anaconda.org/conda-forge/pyarrow) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/pyarrow.svg)](https://anaconda.org/conda-forge/pyarrow) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/pyarrow.svg)](https://anaconda.org/conda-forge/pyarrow) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/pyarrow.svg)](https://anaconda.org/conda-forge/pyarrow) | Installing arrow-cpp-ext ======================== @@ -171,10 +172,10 @@ Installing `arrow-cpp-ext` from the `conda-forge` channel can be achieved by add conda config --add channels conda-forge ``` -Once the `conda-forge` channel has been enabled, `arrow-cpp, arrow-cpp-proc` can be installed with: +Once the `conda-forge` channel has been enabled, `arrow-cpp, arrow-cpp-proc, pyarrow` can be installed with: ``` -conda install arrow-cpp arrow-cpp-proc +conda install arrow-cpp arrow-cpp-proc pyarrow ``` It is possible to list all of the versions of `arrow-cpp` available on your platform with: diff --git a/recipe/bld-pyarrow.bat b/recipe/bld-pyarrow.bat new file mode 100644 index 000000000..8b4f48e64 --- /dev/null +++ b/recipe/bld-pyarrow.bat @@ -0,0 +1,31 @@ +pushd "%SRC_DIR%"\python + +@rem the symlinks for cmake modules don't work here +del cmake_modules\BuildUtils.cmake +del cmake_modules\SetupCxxFlags.cmake +del cmake_modules\CompilerInfo.cmake +del cmake_modules\FindNumPy.cmake +del cmake_modules\FindPythonLibsNew.cmake +copy /Y "%SRC_DIR%\cpp\cmake_modules\BuildUtils.cmake" cmake_modules\ +copy /Y "%SRC_DIR%\cpp\cmake_modules\SetupCxxFlags.cmake" cmake_modules\ +copy /Y "%SRC_DIR%\cpp\cmake_modules\CompilerInfo.cmake" cmake_modules\ +copy /Y "%SRC_DIR%\cpp\cmake_modules\FindNumPy.cmake" cmake_modules\ +copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\ + +SET ARROW_HOME=%LIBRARY_PREFIX% +SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION% +SET PYARROW_BUILD_TYPE=release +SET PYARROW_WITH_S3=1 +SET PYARROW_WITH_HDFS=1 +SET PYARROW_WITH_DATASET=1 +SET PYARROW_WITH_FLIGHT=1 +SET PYARROW_WITH_GANDIVA=1 +SET PYARROW_WITH_PARQUET=1 +SET PYARROW_CMAKE_GENERATOR=Ninja + +%PYTHON% setup.py ^ + build_ext ^ + install --single-version-externally-managed ^ + --record=record.txt +if errorlevel 1 exit 1 +popd diff --git a/recipe/build-pyarrow.sh b/recipe/build-pyarrow.sh new file mode 100644 index 000000000..99396d849 --- /dev/null +++ b/recipe/build-pyarrow.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +set -e +set -x + +# Build dependencies +export ARROW_HOME=$PREFIX +export PARQUET_HOME=$PREFIX +export SETUPTOOLS_SCM_PRETEND_VERSION=$PKG_VERSION +export PYARROW_BUILD_TYPE=release +export PYARROW_WITH_DATASET=1 +export PYARROW_WITH_FLIGHT=1 +export PYARROW_WITH_GANDIVA=1 +export PYARROW_WITH_HDFS=1 +export PYARROW_WITH_ORC=1 +export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_PLASMA=1 +export PYARROW_WITH_S3=1 +BUILD_EXT_FLAGS="" + +# Enable CUDA support +if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]] +then + export PYARROW_WITH_CUDA=1 + BUILD_EXT_FLAGS="${BUILD_EXT_FLAGS} --with-cuda" +else + export PYARROW_WITH_CUDA=0 +fi + +# Resolve: Make Error at cmake_modules/SetupCxxFlags.cmake:338 (message): Unsupported arch flag: -march=. +if [[ "$(uname -m)" = "aarch64" ]] +then + export PYARROW_CMAKE_OPTIONS="-DARROW_ARMV8_ARCH=armv8-a" +fi + +cd python + +$PYTHON setup.py \ + build_ext $BUILD_EXT_FLAGS \ + install --single-version-externally-managed \ + --record=record.txt diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 5262cb457..8a0edf709 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,8 +1,9 @@ {% set version = "0.17.1" %} -{% set number = "2" %} +{% set number = "3" %} {% set cuda_enabled = cuda_compiler_version is not undefined and cuda_compiler_version == '9.2' %} {% set build_ext_version = "1.0.0" %} {% set build_ext = "cuda" if cuda_enabled else "cpu" %} +{% set build_string = "py{}h{}_{}_{}".format(CONDA_PY, PKG_HASH, number, build_ext) %} package: name: arrow-cpp-ext @@ -40,7 +41,7 @@ outputs: version: {{ version }} build: number: {{ number }} - string: "py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ number }}_{{ build_ext }}" + string: "{{ build_string }}" skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] run_exports: - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }} @@ -56,7 +57,7 @@ outputs: - make # [unix] - {{ compiler('c') }} - {{ compiler('cxx') }} - - {{ compiler("cuda") }} # [cuda_compiler_version is not undefined and cuda_compiler_version == '9.2'] + - {{ compiler("cuda") }} # [cuda_compiler_version not in (undefined, "None")] host: - aws-sdk-cpp - boost-cpp @@ -79,7 +80,6 @@ outputs: - thrift-cpp >=0.11 - zlib - zstd - run: - {{ pin_compatible('numpy', lower_bound='1.16') }} - aws-sdk-cpp @@ -97,7 +97,14 @@ outputs: - zstd run_constrained: - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 + - cudatoolkit >=9.2 # [cuda_compiler_version not in (undefined, "None")] + + about: + home: http://github.com/apache/arrow + license: Apache-2.0 + license_file: + - LICENSE.txt + summary: C++ libraries for Apache Arrow test: commands: @@ -150,19 +157,78 @@ outputs: - if exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win] - if exist %PREFIX%\\Library\\lib\\gandiva_static.lib exit 1 # [win] + - name: pyarrow + script: build-pyarrow.sh # [not win] + script: bld-pyarrow.bat # [win] + version: {{ version }} + build: + number: {{ number }} + string: "{{ build_string }}" + skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] + track_features: + {{ "- arrow-cuda" if cuda_enabled else "" }} + requirements: + build: + - cmake 3.16.* + - ninja + - make # [unix] + - {{ compiler('c') }} + - {{ compiler('cxx') }} + # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda + - {{ compiler("cuda") }} # [cuda_compiler_version not in (undefined, "None")] + host: + - {{ pin_subpackage('arrow-cpp', exact=True) }} + - boost-cpp + - cython + - numpy 1.16.* + - python + - setuptools + - setuptools_scm + - six + run: + - {{ pin_subpackage('arrow-cpp', exact=True) }} + - {{ pin_compatible('numpy', lower_bound='1.16') }} + - boost-cpp + - glog + - pandas + # empty parquet-cpp metapackage, force old versions to be uninstalled + - parquet-cpp 1.5.1.* + - python + - six + - snappy + run_constrained: + - arrow-cpp-proc * {{ build_ext }} + - cudatoolkit >=9.2 # [cuda_compiler_version not in (undefined, "None")] + about: home: http://github.com/apache/arrow license: Apache-2.0 license_file: - LICENSE.txt - summary: C++ libraries for Apache Arrow + summary: Python libraries for Apache Arrow + + test: + imports: + - pyarrow + - pyarrow.dataset + - pyarrow.flight + - pyarrow.gandiva + - pyarrow.orc # [unix] + - pyarrow.parquet + - pyarrow.plasma # [unix] + - pyarrow.fs + - pyarrow._s3fs + - pyarrow._hdfs + # We can only test importing cuda package but cannot run when a + # CUDA device is not available, for instance, when building from CI. + - pyarrow.cuda # [cuda_compiler_version not in (undefined, "None")] about: home: http://github.com/apache/arrow license: Apache-2.0 license_file: - LICENSE.txt - summary: C++ libraries for Apache Arrow + summary: C++ and Python libraries for Apache Arrow extra: recipe-maintainers: