From e9c4cd6455c71e3724b789cd9c4a59bbf1205bbf Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 4 Jun 2019 18:28:30 -0400 Subject: [PATCH 1/9] Enable Flight in Conda packages --- dev/tasks/conda-recipes/arrow-cpp/build.sh | 1 + dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 9 +++++++++ dev/tasks/conda-recipes/pyarrow/build.sh | 1 + dev/tasks/conda-recipes/pyarrow/meta.yaml | 1 + 4 files changed, 12 insertions(+) diff --git a/dev/tasks/conda-recipes/arrow-cpp/build.sh b/dev/tasks/conda-recipes/arrow-cpp/build.sh index b18862a83fd..4285e91564d 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build.sh @@ -25,6 +25,7 @@ cmake \ -DARROW_BUILD_UTILITIES=OFF \ -DARROW_BUILD_TESTS=OFF \ -DARROW_JEMALLOC=ON \ + -DARROW_FLIGHT=ON \ -DARROW_PLASMA=ON \ -DARROW_PYTHON=ON \ -DARROW_PARQUET=ON \ diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 877bfc6021c..3fd43f3c859 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -22,10 +22,12 @@ requirements: host: - boost-cpp - brotli + - c-ares - double-conversion - flatbuffers - gflags - glog + - grpc-cpp - libprotobuf - clangdev 7.* - llvmdev 7.* @@ -44,9 +46,11 @@ requirements: - {{ pin_compatible('numpy', lower_bound='1.14') }} - boost-cpp - brotli + - c-ares - double-conversion - gflags - glog + - grpc-cpp - lz4-c - python - re2 @@ -58,6 +62,7 @@ test: commands: # headers - test -f $PREFIX/include/arrow/api.h # [unix] + - test -f $PREFIX/include/arrow/flight/types.h # [unix] - test -f $PREFIX/include/plasma/client.h # [unix] - test -f $PREFIX/include/gandiva/engine.h # [unix] - test -f $PREFIX/include/parquet/api/reader.h # [unix] @@ -67,17 +72,20 @@ test: # static - test -f $PREFIX/lib/libarrow.a # [unix] + - test -f $PREFIX/lib/libarrow_flight.a # [unix] - test -f $PREFIX/lib/libarrow_python.a # [unix] - test -f $PREFIX/lib/libplasma.a # [unix] - test -f $PREFIX/lib/libparquet.a # [unix] - test -f $PREFIX/lib/libgandiva.a # [unix] - if not exist %PREFIX%\\Library\\lib\\arrow_static.lib exit 1 # [win] + - if not exist %PREFIX%\\Library\\lib\\arrow_flight_static.lib exit 1 # [win] - if not exist %PREFIX%\\Library\\lib\\arrow_python_static.lib exit 1 # [win] - if not exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win] - if not exist %PREFIX%\\Library\\lib\\gandiva_static.lib exit 1 # [win] # shared - test -f $PREFIX/lib/libarrow.so # [linux] + - test -f $PREFIX/lib/libarrow_flight.so # [linux] - test -f $PREFIX/lib/libarrow_python.so # [linux] - test -f $PREFIX/lib/libparquet.so # [linux] - test -f $PREFIX/lib/libgandiva.so # [linux] @@ -88,6 +96,7 @@ test: - test -f $PREFIX/lib/libparquet.dylib # [osx] - test -f $PREFIX/lib/libplasma.dylib # [osx] - if not exist %PREFIX%\\Library\\bin\\arrow.dll exit 1 # [win] + - if not exist %PREFIX%\\Library\\bin\\arrow_flight.dll exit 1 # [win] - if not exist %PREFIX%\\Library\\bin\\arrow_python.dll exit 1 # [win] - if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1 # [win] - if not exist %PREFIX%\\Library\\bin\\gandiva.dll exit 1 # [win] diff --git a/dev/tasks/conda-recipes/pyarrow/build.sh b/dev/tasks/conda-recipes/pyarrow/build.sh index 98b10a42e59..161546febe1 100644 --- a/dev/tasks/conda-recipes/pyarrow/build.sh +++ b/dev/tasks/conda-recipes/pyarrow/build.sh @@ -12,6 +12,7 @@ cd python $PYTHON setup.py \ build_ext --build-type=release \ + --with-flight \ --with-orc \ --with-plasma \ --with-parquet \ diff --git a/dev/tasks/conda-recipes/pyarrow/meta.yaml b/dev/tasks/conda-recipes/pyarrow/meta.yaml index 8198390ac53..33f531f0f25 100644 --- a/dev/tasks/conda-recipes/pyarrow/meta.yaml +++ b/dev/tasks/conda-recipes/pyarrow/meta.yaml @@ -45,6 +45,7 @@ requirements: test: imports: - pyarrow + - pyarrow.flight # [py>=35] - pyarrow.plasma # [unix] - pyarrow.parquet - pyarrow.gandiva # [py>=35] From d809fdf2e4e682532936611c4801c661ab01268f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 26 Jun 2019 16:04:46 +0200 Subject: [PATCH 2/9] bundle uriparser --- dev/tasks/conda-recipes/arrow-cpp/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/conda-recipes/arrow-cpp/build.sh b/dev/tasks/conda-recipes/arrow-cpp/build.sh index 4285e91564d..0c4414d3782 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build.sh @@ -19,6 +19,7 @@ cmake \ -DCMAKE_INSTALL_LIBDIR=$PREFIX/lib \ -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \ -DARROW_DEPENDENCY_SOURCE=SYSTEM \ + -Duriparser_SOURCE=BUNDLED \ -DARROW_PACKAGE_PREFIX=$PREFIX \ -DARROW_BOOST_USE_SHARED=ON \ -DARROW_BUILD_BENCHMARKS=OFF \ From 8765af1495bc0c44b097abdc8d18896185224a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 26 Jun 2019 16:48:11 +0200 Subject: [PATCH 3/9] pass ARROW_TEST_DATA --- dev/tasks/conda-recipes/appveyor.win.yml | 1 + dev/tasks/conda-recipes/azure.linux.yml | 1 + dev/tasks/conda-recipes/azure.osx.yml | 1 + dev/tasks/conda-recipes/azure.win.yml | 1 + dev/tasks/conda-recipes/pyarrow/meta.yaml | 2 +- 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/appveyor.win.yml b/dev/tasks/conda-recipes/appveyor.win.yml index 121595a6793..48252f271a0 100644 --- a/dev/tasks/conda-recipes/appveyor.win.yml +++ b/dev/tasks/conda-recipes/appveyor.win.yml @@ -29,6 +29,7 @@ test_script: - git clone --no-checkout {{ arrow.remote }} arrow || exit /B - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} || exit /B - git -C arrow checkout FETCH_HEAD || exit /B + - git -C arrow submodule update --init --recursive || exit /B - pushd arrow\dev\tasks\conda-recipes diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index 355f3b18b84..e398dca82f2 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -23,6 +23,7 @@ jobs: git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive displayName: Clone arrow - script: CI=azure ./run_docker_build.sh diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 7ff3570766f..95f76f0fc53 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -30,6 +30,7 @@ jobs: git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive displayName: Clone arrow - script: | diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index 02fc91ae63f..69029088f75 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -52,6 +52,7 @@ jobs: git clone --no-checkout {{ arrow.remote }} arrow git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive displayName: Clone arrow # Configure the VM diff --git a/dev/tasks/conda-recipes/pyarrow/meta.yaml b/dev/tasks/conda-recipes/pyarrow/meta.yaml index 33f531f0f25..953940942f8 100644 --- a/dev/tasks/conda-recipes/pyarrow/meta.yaml +++ b/dev/tasks/conda-recipes/pyarrow/meta.yaml @@ -55,7 +55,7 @@ test: - hypothesis commands: - - pytest --pyargs pyarrow + - ARROW_TEST_DATA=../../../../testing/data pytest --pyargs pyarrow about: home: http://github.com/apache/arrow From f61250852b8d6cdc2e3a9b1cc2d6e15b66bcf19e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 27 Jun 2019 12:44:10 +0200 Subject: [PATCH 4/9] export ARROW_TEST_DATA on the template level --- dev/tasks/conda-recipes/azure.linux.yml | 1 + dev/tasks/conda-recipes/azure.osx.yml | 1 + dev/tasks/conda-recipes/azure.win.yml | 1 + dev/tasks/conda-recipes/pyarrow/meta.yaml | 2 +- 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index e398dca82f2..8864fa3a6c0 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -8,6 +8,7 @@ jobs: CONFIG: {{ config }} ARROW_VERSION: {{ arrow.no_rc_version }} DOCKER_IMAGE: condaforge/linux-anvil-comp7 + ARROW_TEST_DATA: $(Agent.BuildDirectory)/arrow/testing/data UPLOAD_PACKAGES: False steps: diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 95f76f0fc53..f4f63f05855 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -6,6 +6,7 @@ jobs: variables: CONFIG: {{ config }} ARROW_VERSION: {{ arrow.no_rc_version }} + ARROW_TEST_DATA: $(Agent.BuildDirectory)/arrow/testing/data UPLOAD_PACKAGES: False steps: - script: | diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index 69029088f75..67d25832f57 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -7,6 +7,7 @@ jobs: CONFIG: {{ config }} ARROW_VERSION: {{ arrow.no_rc_version }} CONDA_BLD_PATH: D:\\bld\\ + ARROW_TEST_DATA: $(Agent.BuildDirectory)/arrow/testing/data UPLOAD_PACKAGES: False steps: - script: | diff --git a/dev/tasks/conda-recipes/pyarrow/meta.yaml b/dev/tasks/conda-recipes/pyarrow/meta.yaml index 953940942f8..33f531f0f25 100644 --- a/dev/tasks/conda-recipes/pyarrow/meta.yaml +++ b/dev/tasks/conda-recipes/pyarrow/meta.yaml @@ -55,7 +55,7 @@ test: - hypothesis commands: - - ARROW_TEST_DATA=../../../../testing/data pytest --pyargs pyarrow + - pytest --pyargs pyarrow about: home: http://github.com/apache/arrow From e764a59e1213ab7344b6ff665c13a9c8bb2e4a17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 27 Jun 2019 13:18:09 +0200 Subject: [PATCH 5/9] use SRC_DIR --- dev/tasks/conda-recipes/azure.linux.yml | 1 - dev/tasks/conda-recipes/azure.osx.yml | 1 - dev/tasks/conda-recipes/azure.win.yml | 1 - dev/tasks/conda-recipes/pyarrow/meta.yaml | 1 + 4 files changed, 1 insertion(+), 3 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index 8864fa3a6c0..e398dca82f2 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -8,7 +8,6 @@ jobs: CONFIG: {{ config }} ARROW_VERSION: {{ arrow.no_rc_version }} DOCKER_IMAGE: condaforge/linux-anvil-comp7 - ARROW_TEST_DATA: $(Agent.BuildDirectory)/arrow/testing/data UPLOAD_PACKAGES: False steps: diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index f4f63f05855..95f76f0fc53 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -6,7 +6,6 @@ jobs: variables: CONFIG: {{ config }} ARROW_VERSION: {{ arrow.no_rc_version }} - ARROW_TEST_DATA: $(Agent.BuildDirectory)/arrow/testing/data UPLOAD_PACKAGES: False steps: - script: | diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index 67d25832f57..69029088f75 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -7,7 +7,6 @@ jobs: CONFIG: {{ config }} ARROW_VERSION: {{ arrow.no_rc_version }} CONDA_BLD_PATH: D:\\bld\\ - ARROW_TEST_DATA: $(Agent.BuildDirectory)/arrow/testing/data UPLOAD_PACKAGES: False steps: - script: | diff --git a/dev/tasks/conda-recipes/pyarrow/meta.yaml b/dev/tasks/conda-recipes/pyarrow/meta.yaml index 33f531f0f25..e9739450d5d 100644 --- a/dev/tasks/conda-recipes/pyarrow/meta.yaml +++ b/dev/tasks/conda-recipes/pyarrow/meta.yaml @@ -55,6 +55,7 @@ test: - hypothesis commands: + - export ARROW_TEST_DATA="${SRC_DIR}/testing/data" - pytest --pyargs pyarrow about: From c0b95ac480780067f0ab7bc0b00eabd987673a47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 27 Jun 2019 15:44:10 +0200 Subject: [PATCH 6/9] debug env vars --- dev/tasks/conda-recipes/pyarrow/meta.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/pyarrow/meta.yaml b/dev/tasks/conda-recipes/pyarrow/meta.yaml index e9739450d5d..f2e26b4bb4e 100644 --- a/dev/tasks/conda-recipes/pyarrow/meta.yaml +++ b/dev/tasks/conda-recipes/pyarrow/meta.yaml @@ -55,7 +55,10 @@ test: - hypothesis commands: - - export ARROW_TEST_DATA="${SRC_DIR}/testing/data" + - env + - echo $SRC_DIR + - export ARROW_TEST_DATA=$SRC_DIR/testing/data + - echo $ARROW_TEST_DATA - pytest --pyargs pyarrow about: From c1baa4c430ecbc43db91ba1d16754eedc58fd8ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 27 Jun 2019 16:32:25 +0200 Subject: [PATCH 7/9] skip tests instead --- dev/tasks/conda-recipes/pyarrow/meta.yaml | 6 +----- python/pyarrow/tests/test_flight.py | 3 +++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/dev/tasks/conda-recipes/pyarrow/meta.yaml b/dev/tasks/conda-recipes/pyarrow/meta.yaml index f2e26b4bb4e..acd89af8aed 100644 --- a/dev/tasks/conda-recipes/pyarrow/meta.yaml +++ b/dev/tasks/conda-recipes/pyarrow/meta.yaml @@ -55,11 +55,7 @@ test: - hypothesis commands: - - env - - echo $SRC_DIR - - export ARROW_TEST_DATA=$SRC_DIR/testing/data - - echo $ARROW_TEST_DATA - - pytest --pyargs pyarrow + - pytest -m "not requires_testing_data" --pyargs pyarrow about: home: http://github.com/apache/arrow diff --git a/python/pyarrow/tests/test_flight.py b/python/pyarrow/tests/test_flight.py index 3f83a1c9adf..a8973ab7343 100644 --- a/python/pyarrow/tests/test_flight.py +++ b/python/pyarrow/tests/test_flight.py @@ -618,6 +618,7 @@ def test_location_invalid(): @pytest.mark.slow +@pytest.mark.requires_testing_data def test_tls_fails(): """Make sure clients cannot connect when cert verification fails.""" certs = example_tls_certs() @@ -635,6 +636,7 @@ def test_tls_fails(): client.do_get(flight.Ticket(b'ints')) +@pytest.mark.requires_testing_data def test_tls_do_get(): """Try a simple do_get call over TLS.""" table = simple_ints_table() @@ -650,6 +652,7 @@ def test_tls_do_get(): assert data.equals(table) +@pytest.mark.requires_testing_data def test_tls_override_hostname(): """Check that incorrectly overriding the hostname fails.""" certs = example_tls_certs() From 55d2f414ab167baa3cf8826201bd8ccd24c1b426 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 27 Jun 2019 16:30:06 -0500 Subject: [PATCH 8/9] Build Flight in Windows conda packages [skip ci] --- dev/tasks/conda-recipes/arrow-cpp/bld.bat | 1 + dev/tasks/conda-recipes/pyarrow/bld.bat | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld.bat b/dev/tasks/conda-recipes/arrow-cpp/bld.bat index 429cb642d59..f508cd10d07 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld.bat @@ -10,6 +10,7 @@ cmake -G "%CMAKE_GENERATOR%" ^ -DARROW_BUILD_TESTS:BOOL=OFF ^ -DARROW_BUILD_UTILITIES:BOOL=OFF ^ -DCMAKE_BUILD_TYPE=release ^ + -DARROW_FLIGHT:BOOL=ON ^ -DARROW_PYTHON:BOOL=ON ^ -DARROW_PARQUET:BOOL=ON ^ -DARROW_GANDIVA:BOOL=ON ^ diff --git a/dev/tasks/conda-recipes/pyarrow/bld.bat b/dev/tasks/conda-recipes/pyarrow/bld.bat index d02ceaffe48..28eb76a2fd8 100644 --- a/dev/tasks/conda-recipes/pyarrow/bld.bat +++ b/dev/tasks/conda-recipes/pyarrow/bld.bat @@ -14,10 +14,11 @@ copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\ SET ARROW_HOME=%LIBRARY_PREFIX% SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION% +SET PYARROW_WITH_FLIGHT=1 +SET PYARROW_WITH_GANDIVA=1 +SET PYARROW_WITH_PARQUET=1 "%PYTHON%" setup.py ^ build_ext --build-type=release ^ - --with-parquet ^ - --with-gandiva ^ install --single-version-externally-managed ^ --record=record.txt if errorlevel 1 exit 1 From 909b798c26ca4cd3758b034b98115d3718b5319c Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 27 Jun 2019 16:31:51 -0500 Subject: [PATCH 9/9] Also build uriparser bundled in windows [skip ci] --- dev/tasks/conda-recipes/arrow-cpp/bld.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld.bat b/dev/tasks/conda-recipes/arrow-cpp/bld.bat index f508cd10d07..d3ea1b24690 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld.bat @@ -15,6 +15,7 @@ cmake -G "%CMAKE_GENERATOR%" ^ -DARROW_PARQUET:BOOL=ON ^ -DARROW_GANDIVA:BOOL=ON ^ -DARROW_ORC:BOOL=ON ^ + -Duriparser_SOURCE=BUNDLED ^ .. cmake --build . --target INSTALL --config Release