diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml index 31c61b17fc0..5d80a17c4df 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml @@ -1,5 +1,7 @@ +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -31,11 +33,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -48,7 +50,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -59,13 +61,15 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - linux-64 thrift_cpp: -- 0.16.0 +- 0.18.1 +ucx: +- 1.14.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml index e0629f86278..39b25b44690 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml @@ -1,5 +1,7 @@ +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -31,11 +33,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -48,7 +50,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -59,13 +61,15 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - linux-64 thrift_cpp: -- 0.16.0 +- 0.18.1 +ucx: +- 1.14.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml index 793c1afa7ab..af0fc2dcd25 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml @@ -1,7 +1,9 @@ BUILD: - aarch64-conda_cos7-linux-gnu +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -31,11 +33,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -48,7 +50,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -59,13 +61,15 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - linux-aarch64 thrift_cpp: -- 0.16.0 +- 0.18.1 +ucx: +- 1.14.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml similarity index 88% rename from dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml index 6c81ae03b6d..83a1f7f7400 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml @@ -1,5 +1,7 @@ +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -27,11 +29,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -44,7 +46,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -55,13 +57,15 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - linux-ppc64le thrift_cpp: -- 0.16.0 +- 0.18.1 +ucx: +- 1.14.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_64_.yaml index bff4181d549..0cf990cc113 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml @@ -1,7 +1,9 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -25,11 +27,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -44,7 +46,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -55,13 +57,13 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - osx-64 thrift_cpp: -- 0.16.0 +- 0.18.1 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml index 2e9a5682b95..3faa6278e81 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml @@ -1,7 +1,9 @@ MACOSX_DEPLOYMENT_TARGET: - '11.0' +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -25,11 +27,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -44,7 +46,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -55,13 +57,13 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - osx-arm64 thrift_cpp: -- 0.16.0 +- 0.18.1 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml deleted file mode 100644 index 96784cc8b6f..00000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml +++ /dev/null @@ -1,27 +0,0 @@ -c_compiler: -- gcc -c_compiler_version: -- '11' -cdt_name: -- cos6 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '11' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.1' -target_platform: -- linux-64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml deleted file mode 100644 index 413d8c6583f..00000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml +++ /dev/null @@ -1,31 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -c_compiler: -- gcc -c_compiler_version: -- '11' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '11' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.1' -target_platform: -- linux-aarch64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml deleted file mode 100644 index 7e5b3d08039..00000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml +++ /dev/null @@ -1,27 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -c_compiler: -- clang -c_compiler_version: -- '14' -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- clangxx -cxx_compiler_version: -- '14' -macos_machine: -- x86_64-apple-darwin13.4.0 -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.1' -target_platform: -- osx-64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml deleted file mode 100644 index 5c58110332f..00000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml +++ /dev/null @@ -1,27 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '11.0' -c_compiler: -- clang -c_compiler_version: -- '14' -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- clangxx -cxx_compiler_version: -- '14' -macos_machine: -- arm64-apple-darwin20.0.0 -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.1' -target_platform: -- osx-arm64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml similarity index 90% rename from dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml index 7173299488e..6ea00e3bd0d 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -23,15 +23,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libcrc32c: - '1.1' libcurl: - '7' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -43,6 +43,8 @@ numpy: - '1.20' openssl: - '3' +orc: +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -53,13 +55,13 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - win-64 thrift_cpp: -- 0.16.0 +- 0.18.1 zip_keys: - - python - numpy diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml similarity index 90% rename from dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml index 49fc4defc1b..183356662c6 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -23,15 +23,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libcrc32c: - '1.1' libcurl: - '7' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -43,6 +43,8 @@ numpy: - '1.20' openssl: - '3' +orc: +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -53,13 +55,13 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - win-64 thrift_cpp: -- 0.16.0 +- 0.18.1 zip_keys: - - python - numpy diff --git a/dev/tasks/conda-recipes/arrow-cpp/activate.sh b/dev/tasks/conda-recipes/arrow-cpp/activate.sh new file mode 100644 index 00000000000..90210fac0a0 --- /dev/null +++ b/dev/tasks/conda-recipes/arrow-cpp/activate.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# for the gdb-wrappers, we need to create a symlink that +# contains the full path of the lib _within_ the installed +# env, which we don't have until the env is created. + +# doesn't come with a deactivate script, because the symlink +# is benign and doesn't need to be deleted. + +# where the GDB wrappers get installed +GDB_PREFIX=$CONDA_PREFIX/share/gdb/auto-load + +# If the directory is not writable, nothing can be done +if [ ! -w $GDB_PREFIX ]; then + return +fi + +# this needs to be in sync with the respective patch +PLACEHOLDER=replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX +# the paths here are intentionally stacked, see #935, resp. +# https://github.com/apache/arrow/blob/master/docs/source/cpp/gdb.rst#manual-loading +WRAPPER_DIR=$GDB_PREFIX/$CONDA_PREFIX/lib + +mkdir -p $WRAPPER_DIR +# there's only one lib in that folder, but the libname changes +# based on the version so use a loop instead of hardcoding it. +for f in $GDB_PREFIX/$PLACEHOLDER/lib/*.py; do + # overwrite, because we don't have deactivation (i.e. symlink remains) + ln -sf $f $WRAPPER_DIR/$(basename $f) +done diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat index b3c283ddb8f..60c81be7411 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat @@ -14,6 +14,11 @@ if "%cuda_compiler_version%"=="None" ( set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=ON" ) +:: # reusable variable for dependencies we cannot yet unvendor +set "READ_RECIPE_META_YAML_WHY_NOT=OFF" + +:: for available switches see +:: https://github.com/apache/arrow/blame/apache-arrow-11.0.0/cpp/cmake_modules/DefineOptions.cmake cmake -G "Ninja" ^ -DARROW_BOOST_USE_SHARED:BOOL=ON ^ -DARROW_BUILD_STATIC:BOOL=OFF ^ @@ -32,7 +37,7 @@ cmake -G "Ninja" ^ -DARROW_HDFS:BOOL=ON ^ -DARROW_JSON:BOOL=ON ^ -DARROW_MIMALLOC:BOOL=ON ^ - -DARROW_ORC:BOOL=OFF ^ + -DARROW_ORC:BOOL=ON ^ -DARROW_PACKAGE_PREFIX="%LIBRARY_PREFIX%" ^ -DARROW_PARQUET:BOOL=ON ^ -DARROW_S3:BOOL=ON ^ @@ -42,6 +47,8 @@ cmake -G "Ninja" ^ -DARROW_WITH_BROTLI:BOOL=ON ^ -DARROW_WITH_BZ2:BOOL=ON ^ -DARROW_WITH_LZ4:BOOL=ON ^ + -DARROW_WITH_NLOHMANN_JSON:BOOL=ON ^ + -DARROW_WITH_OPENTELEMETRY:BOOL=%READ_RECIPE_META_YAML_WHY_NOT% ^ -DARROW_WITH_SNAPPY:BOOL=ON ^ -DARROW_WITH_ZLIB:BOOL=ON ^ -DARROW_WITH_ZSTD:BOOL=ON ^ @@ -56,9 +63,9 @@ cmake -G "Ninja" ^ -DPython3_EXECUTABLE="%PYTHON%" ^ %EXTRA_CMAKE_ARGS% ^ .. -if errorlevel 1 exit 1 +if %ERRORLEVEL% neq 0 exit 1 cmake --build . --target install --config Release -if errorlevel 1 exit 1 +if %ERRORLEVEL% neq 0 exit 1 popd diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat index ed0e20554b7..084faf74e4a 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat @@ -1,4 +1,5 @@ @echo on + pushd "%SRC_DIR%"\python @rem the symlinks for cmake modules don't work here @@ -39,7 +40,7 @@ if "%cuda_compiler_version%"=="None" ( build_ext ^ install --single-version-externally-managed ^ --record=record.txt -if errorlevel 1 exit 1 +if %ERRORLEVEL% neq 0 exit 1 popd if [%PKG_NAME%] == [pyarrow] ( diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index ddeec5b75cf..fb8cbade865 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -1,7 +1,13 @@ #!/bin/bash +set -ex -set -e -set -x +# Copy the [de]activate scripts to $PREFIX/etc/conda/[de]activate.d, see +# https://conda-forge.org/docs/maintainer/adding_pkgs.html#activate-scripts +for CHANGE in "activate" +do + mkdir -p "${PREFIX}/etc/conda/${CHANGE}.d" + cp "${RECIPE_DIR}/${CHANGE}.sh" "${PREFIX}/etc/conda/${CHANGE}.d/${PKG_NAME}_${CHANGE}.sh" +done mkdir cpp/build pushd cpp/build @@ -12,10 +18,13 @@ EXTRA_CMAKE_ARGS="" if [ "$(uname)" == "Linux" ]; then SYSTEM_INCLUDES=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') ARROW_GANDIVA_PC_CXX_FLAGS="${SYSTEM_INCLUDES}" + # only available on linux + ARROW_WITH_UCX=ON else # See https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" ARROW_GANDIVA_PC_CXX_FLAGS="-D_LIBCPP_DISABLE_AVAILABILITY" + ARROW_WITH_UCX=OFF fi # Enable CUDA support @@ -55,6 +64,17 @@ if [[ "${target_platform}" == "linux-aarch64" ]] || [[ "${target_platform}" == " export CMAKE_BUILD_PARALLEL_LEVEL=3 fi +# point to a usable protoc if we're running on a different architecture than the target +if [[ "${build_platform}" != "${target_platform}" ]]; then + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc" +fi + +# reusable variable for dependencies we cannot yet unvendor +export READ_RECIPE_META_YAML_WHY_NOT=OFF + +# for available switches see +# https://github.com/apache/arrow/blame/apache-arrow-11.0.0/cpp/cmake_modules/DefineOptions.cmake +# placeholder in ARROW_GDB_INSTALL_DIR must match what's used for replacement in activate.sh cmake -GNinja \ -DARROW_BOOST_USE_SHARED=ON \ -DARROW_BUILD_BENCHMARKS=OFF \ @@ -73,6 +93,7 @@ cmake -GNinja \ -DARROW_GANDIVA=ON \ -DARROW_GANDIVA_PC_CXX_FLAGS="${ARROW_GANDIVA_PC_CXX_FLAGS}" \ -DARROW_GCS=ON \ + -DARROW_GDB_INSTALL_DIR=replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX/lib \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ -DARROW_JSON=ON \ @@ -88,7 +109,10 @@ cmake -GNinja \ -DARROW_WITH_BROTLI=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_NLOHMANN_JSON=ON \ + -DARROW_WITH_OPENTELEMETRY=${READ_RECIPE_META_YAML_WHY_NOT} \ -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_UCX=${ARROW_WITH_UCX} \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ -DBUILD_SHARED_LIBS=ON \ @@ -98,7 +122,6 @@ cmake -GNinja \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \ -DPARQUET_REQUIRE_ENCRYPTION=ON \ - -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc \ -DPython3_EXECUTABLE=${PYTHON} \ ${EXTRA_CMAKE_ARGS} \ .. diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh index 5d5d98d1c8d..14c67ede632 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -1,7 +1,5 @@ #!/bin/sh - -set -e -set -x +set -ex # Build dependencies export ARROW_HOME=$PREFIX diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 3aa30a65025..2f79bbe958c 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -6,6 +6,9 @@ {% set proc_build_number = "0" %} {% set llvm_version = "14" %} +# see https://github.com/apache/arrow/blob/apache-arrow-10.0.1/cpp/CMakeLists.txt#L88-L90 +{% set so_version = (version.split(".")[0] | int * 100 + version.split(".")[1] | int) ~ "." ~ version.split(".")[2] ~ ".0" %} + package: name: apache-arrow version: {{ version }} @@ -30,12 +33,6 @@ outputs: build: number: {{ proc_build_number }} string: {{ build_ext }} - ignore_run_exports_from: - - openssl - requirements: - host: - # ensure we split the CI jobs per openssl version - - openssl test: commands: - exit 0 @@ -52,12 +49,7 @@ outputs: build: number: {{ proc_build_number }} string: {{ build_ext }} - ignore_run_exports_from: - - openssl requirements: - host: - # ensure we split the CI jobs per openssl version - - openssl run: - apache-arrow-proc ={{ build_ext_version }}={{ build_ext }} test: @@ -80,29 +72,39 @@ outputs: - "*/nvcuda.dll" # [win] requirements: build: + - {{ compiler("c") }} + - {{ compiler("cxx") }} + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - clangdev {{ llvm_version }} # [osx and arm64] - llvmdev {{ llvm_version }} # [osx and arm64] - gnuconfig # [osx and arm64] - - libgrpc - - libprotobuf + # needs to run protoc & grpc_cpp_plugin + - libgrpc # [build_platform != target_platform] + - libprotobuf # [build_platform != target_platform] - cmake - - autoconf # [unix] - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] + # necessary for vendored jemalloc + - autoconf # [linux] + - make # [linux] host: + # for required dependencies, see + # https://github.com/apache/arrow/blob/apache-arrow-11.0.0/cpp/cmake_modules/ThirdpartyToolchain.cmake#L46-L75 - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} + - aws-crt-cpp # [unix] - aws-sdk-cpp - boost-cpp >=1.70 - brotli - bzip2 - c-ares + # not yet: https://github.com/conda-forge/cpp-opentelemetry-sdk-feedstock/issues/38 + # - cpp-opentelemetry-sdk + # - proto-opentelemetry-proto =={{ cpp_opentelemetry_sdk }} - gflags - glog - google-cloud-cpp + # arrow uses a customized jemalloc, see #944 + # - jemalloc - libabseil # since libgoogle-cloud is static on windows, see # https://github.com/conda-forge/google-cloud-cpp-feedstock/pull/108, @@ -113,13 +115,15 @@ outputs: - libprotobuf - libutf8proc - lz4-c + - nlohmann_json # gandiva depends on openssl - openssl - - orc # [unix] + - orc - rapidjson - re2 - snappy - thrift-cpp + - ucx # [linux] - xsimd - zlib - zstd @@ -164,6 +168,10 @@ outputs: - if exist %LIBRARY_LIB%\arrow_cuda.lib exit 1 # [(cuda_compiler_version == "None") and win] - if exist %LIBRARY_LIB%\arrow_cuda_static.lib exit 1 # [(cuda_compiler_version == "None") and win] + # gdb-wrapper (paths are stacked intentionally) + - test -f $PREFIX/share/gdb/auto-load/$PREFIX/lib/libarrow.so.{{ so_version }}-gdb.py # [linux] + - test -f $PREFIX/share/gdb/auto-load/$PREFIX/lib/libarrow.{{ so_version }}.dylib-gdb.py # [osx] + about: home: http://github.com/apache/arrow license: Apache-2.0 @@ -180,8 +188,6 @@ outputs: - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} requirements: host: - # make sure the builds for different openssl versions (with the same PKG_BUILDNUM) have different hashes - - openssl - {{ pin_subpackage('libarrow', exact=True) }} run: - {{ pin_subpackage('libarrow', exact=True) }} @@ -197,30 +203,26 @@ outputs: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} ignore_run_exports: - cudatoolkit - ignore_run_exports_from: - - openssl track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} + rpaths: + - lib/ + - {{ SP_DIR }}/pyarrow missing_dso_whitelist: # not actually missing, but installed into SP_DIR, see tests - - "*/libarrow_python.so.*" # [linux] - - "*/libarrow_python_flight.so.*" # [linux] - - "*/libarrow_python.*.dylib" # [osx] - - "*/libarrow_python_flight.*.dylib" # [osx] - "*/arrow_python.dll" # [win] - "*/arrow_python_flight.dll" # [win] requirements: build: + - {{ compiler("c") }} + - {{ compiler("cxx") }} + # pyarrow does not require nvcc but it needs to link against libraries in libarrow=*=*cuda + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - cython # [build_platform != target_platform] - numpy # [build_platform != target_platform] - cmake - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - # pyarrow does not require nvcc but it needs to link against libraries in libarrow=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] host: - {{ pin_subpackage('libarrow', exact=True) }} - clangdev {{ llvm_version }} @@ -228,7 +230,6 @@ outputs: - cython - gflags # [unix] - numpy - - openssl - python - setuptools - setuptools_scm @@ -294,22 +295,19 @@ outputs: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} ignore_run_exports: - cudatoolkit - ignore_run_exports_from: - - openssl track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} requirements: build: + - {{ compiler("c") }} + - {{ compiler("cxx") }} + # pyarrow does not require nvcc but it needs to link against libraries in libarrow=*=*cuda + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - cython # [build_platform != target_platform] - numpy # [build_platform != target_platform] - cmake - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - # pyarrow does not require nvcc but it needs to link against libraries in libarrow=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] host: - {{ pin_subpackage('libarrow', exact=True) }} - {{ pin_subpackage('pyarrow', exact=True) }} @@ -317,7 +315,6 @@ outputs: - llvmdev {{ llvm_version }} - cython - numpy - - openssl - python - setuptools - setuptools_scm @@ -330,9 +327,11 @@ outputs: test: requires: + # test_cpp_extension_in_python requires a compiler + - {{ compiler("cxx") }} # [linux] - pytest - pytest-lazy-fixture - - backports.zoneinfo # [py<39] + - backports.zoneinfo # [py<39] - cffi - cloudpickle - cython @@ -340,17 +339,17 @@ outputs: - fsspec - hypothesis - pandas - # not all arches available in conda-forge; osx-arm not tested anyway - - pytorch * # [unix and not ppc64le and py<311] - scipy + # not all OSes/arches available in conda-forge + - pytorch * # [unix and not ppc64le] + # not yet rebuilt for libabseil 20230125 + # - tensorflow # [unix and x86_64 and py<311] # we're not building java bindings # - jpype1 # doesn't get picked up correctly # - libhdfs3 # causes segfaults # - sparse - # not rebuilt for grpc 1.49 yet - # - tensorflow source_files: - testing/data commands: @@ -365,6 +364,8 @@ outputs: # skip tests that raise SIGINT and crash the test suite {% set tests_to_skip = tests_to_skip + " or (test_csv and test_cancellation)" %} # [linux] {% set tests_to_skip = tests_to_skip + " or (test_flight and test_interrupt)" %} # [linux] + # cannot pass -D_LIBCPP_DISABLE_AVAILABILITY to test suite for our older macos sdk + {% set tests_to_skip = tests_to_skip + " or test_cpp_extension_in_python" %} # [osx] # skip tests that make invalid(-for-conda) assumptions about the compilers setup {% set tests_to_skip = tests_to_skip + " or test_cython_api" %} # [unix] {% set tests_to_skip = tests_to_skip + " or test_visit_strings" %} # [unix] @@ -372,6 +373,14 @@ outputs: {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool_disabled" %} # [aarch64 or ppc64le] {% set tests_to_skip = tests_to_skip + " or test_env_var_io_thread_count" %} # [aarch64 or ppc64le] # vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv + # segfaults on OSX: to investigate ASAP + {% set tests_to_skip = tests_to_skip + " or test_flight" %} # [osx] + # gandiva tests are segfaulting on ppc + {% set tests_to_skip = tests_to_skip + " or test_gandiva" %} # [ppc64le] + # test failures on ppc + {% set tests_to_skip = tests_to_skip + " or test_safe_cast_from_float_with_nans_to_int" %} # [ppc64le] + # gandiva tests are segfaulting on ppc + {% set tests_to_skip = tests_to_skip + " or test_float_with_null_as_integer" %} # [ppc64le] # "Unsupported backend 'nonexistent' specified in ARROW_DEFAULT_MEMORY_POOL" {% set tests_to_skip = tests_to_skip + " or (test_memory and test_env_var)" %} # [unix] # test is broken; header is in $PREFIX, not $SP_DIR @@ -379,9 +388,7 @@ outputs: # ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^ # crossbow CI: reduce to one python version, except on (unemulated) linux, where it's fast enough - - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [linux64 or ((py==310) and not ppc64le)] - # NOTE: test suite is skipped on ppc due to bugs in QEMU code that cause - # CI to fail, even though the tests would run through on native hardware + - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [linux64 or (py==310 and build_platform==target_platform)] about: home: http://github.com/apache/arrow diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index 95c5fc10056..ee4533a3ce7 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -3,7 +3,7 @@ jobs: - job: win pool: - vmImage: windows-2019 + vmImage: windows-2022 timeoutInMinutes: 360 variables: CONFIG: {{ config }} @@ -32,7 +32,7 @@ jobs: - script: | call activate base - mamba.exe install 'python=3.9' conda-build conda pip boa 'conda-forge-ci-setup=3' -c conda-forge --strict-channel-priority --yes + mamba.exe install 'python=3.10' conda-build conda pip boa 'conda-forge-ci-setup=3' -c conda-forge --strict-channel-priority --yes displayName: Install conda-build - script: set PYTHONUNBUFFERED=1 diff --git a/dev/tasks/conda-recipes/r-arrow/meta.yaml b/dev/tasks/conda-recipes/r-arrow/meta.yaml index 3d053a781b0..4c86dc9280c 100644 --- a/dev/tasks/conda-recipes/r-arrow/meta.yaml +++ b/dev/tasks/conda-recipes/r-arrow/meta.yaml @@ -10,6 +10,8 @@ source: path: ../../../../ build: + # 4.1 not usable anymore unless https://github.com/conda-forge/r-base-feedstock/pull/236 gets merged + skip: true # [unix and (r_base == "4.1")] merge_build_host: true # [win] number: 0 rpaths: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 89fbe9b46ad..e6013467803 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -246,21 +246,11 @@ tasks: # generated and to be synced regularly from the feedstock. We have no way # yet to generate them inside the arrow repository automatically. - conda-linux-x64-cpu-r41: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_versionNoneopenssl3 - r_config: linux_64_r_base4.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-linux-x64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_cuda_compiler_versionNoneopenssl3 + config: linux_64_cuda_compiler_versionNone r_config: linux_64_r_base4.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -270,7 +260,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_cuda_compiler_versionNoneopenssl3 + config: linux_64_cuda_compiler_versionNone artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -282,7 +272,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_cuda_compiler_version10.2openssl3 + config: linux_64_cuda_compiler_version10.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda @@ -292,21 +282,11 @@ tasks: ########################### Conda Linux (aarch64) ########################### - conda-linux-aarch64-cpu-r41: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_aarch64_cuda_compiler_versionNoneopenssl3 - r_config: linux_aarch64_r_base4.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-linux-aarch64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_aarch64_cuda_compiler_versionNoneopenssl3 + config: linux_aarch64_cuda_compiler_versionNone r_config: linux_aarch64_r_base4.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -316,7 +296,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_aarch64_cuda_compiler_versionNoneopenssl3 + config: linux_aarch64_cuda_compiler_versionNone artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -330,7 +310,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_ppc64le_cuda_compiler_versionNoneopenssl3 + config: linux_ppc64le_cuda_compiler_versionNone artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -340,21 +320,11 @@ tasks: ############################## Conda OSX (x64) ############################## - conda-osx-x64-cpu-r41: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_64_openssl3 - r_config: osx_64_r_base4.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-osx-x64-cpu-r42: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_64_openssl3 + config: osx_64_ r_config: osx_64_r_base4.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -364,7 +334,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_64_openssl3 + config: osx_64_ artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -374,21 +344,11 @@ tasks: ############################# Conda OSX (arm64) ############################# - conda-osx-arm64-cpu-r41: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_arm64_openssl3 - r_config: osx_arm64_r_base4.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-osx-arm64-cpu-r42: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_arm64_openssl3 + config: osx_arm64_ r_config: osx_arm64_r_base4.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -398,7 +358,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_arm64_openssl3 + config: osx_arm64_ artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -412,7 +372,7 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNoneopenssl3 + config: win_64_cuda_compiler_versionNone r_config: win_64_ artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -424,7 +384,7 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNoneopenssl3 + config: win_64_cuda_compiler_versionNone artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -436,7 +396,7 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNoneopenssl3 + config: win_64_cuda_compiler_versionNone artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda