diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml similarity index 79% rename from dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml index 5d80a17c4df..1cdcec199e7 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml @@ -1,17 +1,15 @@ aws_crt_cpp: -- 0.19.8 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - gcc c_compiler_version: -- '7' +- '10' cdt_name: -- cos6 +- cos7 channel_sources: - conda-forge channel_targets: @@ -19,38 +17,40 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- '10.2' +- '11.2' cuda_compiler_version_min: -- '10.2' +- '11.2' cxx_compiler: - gxx cxx_compiler_version: -- '7' +- '10' docker_image: -- quay.io/condaforge/linux-anvil-cos7-cuda:10.2 +- quay.io/condaforge/linux-anvil-cuda:11.2 gflags: - '2.2' glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- '2.12' libabseil: - '20230125' libgrpc: -- '1.52' +- '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -61,7 +61,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -73,9 +73,12 @@ ucx: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml index 39b25b44690..5be5b58a739 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml @@ -1,15 +1,13 @@ aws_crt_cpp: -- 0.19.8 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - gcc c_compiler_version: -- '11' +- '12' cdt_name: - cos6 channel_sources: @@ -17,15 +15,15 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- nvcc +- None cuda_compiler_version: - None cuda_compiler_version_min: -- '10.2' +- '11.2' cxx_compiler: - gxx cxx_compiler_version: -- '11' +- '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -33,24 +31,26 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- '2.12' libabseil: - '20230125' libgrpc: -- '1.52' +- '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -61,7 +61,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -73,9 +73,12 @@ ucx: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml new file mode 100644 index 00000000000..1677b03564c --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml @@ -0,0 +1,91 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +aws_crt_cpp: +- 0.20.3 +aws_sdk_cpp: +- 1.10.57 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '10' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '11.2' +cuda_compiler_version_min: +- '11.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '10' +docker_image: +- quay.io/condaforge/linux-anvil-cuda:11.2 +gflags: +- '2.2' +glog: +- '0.6' +google_cloud_cpp: +- '2.12' +libabseil: +- '20230125' +libgrpc: +- '1.54' +- '1.56' +libprotobuf: +- '3.21' +- 4.23.3 +lz4_c: +- 1.9.3 +numpy: +- '1.21' +- '1.23' +- '1.21' +- '1.21' +openssl: +- '3' +orc: +- 1.9.0 +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +re2: +- 2023.03.02 +snappy: +- '1' +target_platform: +- linux-aarch64 +thrift_cpp: +- 0.18.1 +ucx: +- 1.14.0 +zip_keys: +- - c_compiler_version + - cxx_compiler_version + - cuda_compiler + - cuda_compiler_version + - cdt_name + - docker_image +- - libgrpc + - libprotobuf +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml index af0fc2dcd25..88fdf1254e6 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml @@ -1,17 +1,15 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_crt_cpp: -- 0.19.8 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - gcc c_compiler_version: -- '11' +- '12' cdt_arch: - aarch64 cdt_name: @@ -20,12 +18,16 @@ channel_sources: - conda-forge channel_targets: - conda-forge main +cuda_compiler: +- None cuda_compiler_version: - None +cuda_compiler_version_min: +- '11.2' cxx_compiler: - gxx cxx_compiler_version: -- '11' +- '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -33,24 +35,26 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- '2.12' libabseil: - '20230125' libgrpc: -- '1.52' +- '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -61,7 +65,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -73,9 +77,12 @@ ucx: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml new file mode 100644 index 00000000000..3585db7b99b --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml @@ -0,0 +1,87 @@ +aws_crt_cpp: +- 0.20.3 +aws_sdk_cpp: +- 1.10.57 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '10' +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '11.2' +cuda_compiler_version_min: +- '11.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '10' +docker_image: +- quay.io/condaforge/linux-anvil-cuda:11.2 +gflags: +- '2.2' +glog: +- '0.6' +google_cloud_cpp: +- '2.12' +libabseil: +- '20230125' +libgrpc: +- '1.54' +- '1.56' +libprotobuf: +- '3.21' +- 4.23.3 +lz4_c: +- 1.9.3 +numpy: +- '1.21' +- '1.23' +- '1.21' +- '1.21' +openssl: +- '3' +orc: +- 1.9.0 +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.10.* *_cpython +- 3.11.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +re2: +- 2023.03.02 +snappy: +- '1' +target_platform: +- linux-ppc64le +thrift_cpp: +- 0.18.1 +ucx: +- 1.14.0 +zip_keys: +- - c_compiler_version + - cxx_compiler_version + - cuda_compiler + - cuda_compiler_version + - cdt_name + - docker_image +- - libgrpc + - libprotobuf +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml index 83a1f7f7400..c13a5222542 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml @@ -1,27 +1,29 @@ aws_crt_cpp: -- 0.19.8 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - gcc c_compiler_version: -- '11' +- '12' cdt_name: - cos7 channel_sources: - conda-forge channel_targets: - conda-forge main +cuda_compiler: +- None cuda_compiler_version: - None +cuda_compiler_version_min: +- '11.2' cxx_compiler: - gxx cxx_compiler_version: -- '11' +- '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 gflags: @@ -29,24 +31,26 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- '2.12' libabseil: - '20230125' libgrpc: -- '1.52' +- '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -57,7 +61,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -69,9 +73,12 @@ ucx: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml index 0cf990cc113..dd4a230760e 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml @@ -1,17 +1,15 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_crt_cpp: -- 0.19.8 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - clang c_compiler_version: -- '14' +- '15' channel_sources: - conda-forge channel_targets: @@ -21,19 +19,21 @@ cuda_compiler_version: cxx_compiler: - clangxx cxx_compiler_version: -- '14' +- '15' gflags: - '2.2' glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- '2.12' libabseil: - '20230125' libgrpc: -- '1.52' +- '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 macos_machine: @@ -41,12 +41,12 @@ macos_machine: numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -57,7 +57,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -67,6 +67,8 @@ thrift_cpp: zip_keys: - - c_compiler_version - cxx_compiler_version +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml index 3faa6278e81..6a6713a54fe 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml @@ -1,17 +1,15 @@ MACOSX_DEPLOYMENT_TARGET: - '11.0' aws_crt_cpp: -- 0.19.8 +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: - '1' -c_ares: -- '1' c_compiler: - clang c_compiler_version: -- '14' +- '15' channel_sources: - conda-forge channel_targets: @@ -21,19 +19,21 @@ cuda_compiler_version: cxx_compiler: - clangxx cxx_compiler_version: -- '14' +- '15' gflags: - '2.2' glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- '2.12' libabseil: - '20230125' libgrpc: -- '1.52' +- '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 macos_machine: @@ -41,12 +41,12 @@ macos_machine: numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -57,7 +57,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -67,6 +67,8 @@ thrift_cpp: zip_keys: - - c_compiler_version - cxx_compiler_version +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml new file mode 100644 index 00000000000..e63767cbe97 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml @@ -0,0 +1,27 @@ +c_compiler: +- gcc +c_compiler_version: +- '12' +cdt_name: +- cos6 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '12' +docker_image: +- quay.io/condaforge/linux-anvil-cos7-x86_64 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- linux-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml index 38753baa7ed..6e661e1357d 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.2.yaml @@ -1,7 +1,7 @@ c_compiler: - gcc c_compiler_version: -- '11' +- '12' cdt_name: - cos6 channel_sources: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '11' +- '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml new file mode 100644 index 00000000000..2b80b020fdc --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml @@ -0,0 +1,31 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +c_compiler: +- gcc +c_compiler_version: +- '12' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '12' +docker_image: +- quay.io/condaforge/linux-anvil-cos7-x86_64 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml index 2913bbb4f14..9dcd0c34c85 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.2.yaml @@ -3,7 +3,7 @@ BUILD: c_compiler: - gcc c_compiler_version: -- '11' +- '12' cdt_arch: - aarch64 cdt_name: @@ -15,7 +15,7 @@ channel_targets: cxx_compiler: - gxx cxx_compiler_version: -- '11' +- '12' docker_image: - quay.io/condaforge/linux-anvil-cos7-x86_64 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml new file mode 100644 index 00000000000..6be6c2f5462 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml @@ -0,0 +1,27 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +c_compiler: +- clang +c_compiler_version: +- '15' +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '15' +macos_machine: +- x86_64-apple-darwin13.4.0 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- osx-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml index 25437ee4adc..2116eaf7b8b 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.2.yaml @@ -3,7 +3,7 @@ MACOSX_DEPLOYMENT_TARGET: c_compiler: - clang c_compiler_version: -- '14' +- '15' channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - clangxx cxx_compiler_version: -- '14' +- '15' macos_machine: - x86_64-apple-darwin13.4.0 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml new file mode 100644 index 00000000000..0ce856fcccf --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml @@ -0,0 +1,27 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +c_compiler: +- clang +c_compiler_version: +- '15' +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '15' +macos_machine: +- arm64-apple-darwin20.0.0 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- osx-arm64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml index 1557b23ff96..af8a07c4220 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.2.yaml @@ -3,7 +3,7 @@ MACOSX_DEPLOYMENT_TARGET: c_compiler: - clang c_compiler_version: -- '14' +- '15' channel_sources: - conda-forge channel_targets: @@ -11,7 +11,7 @@ channel_targets: cxx_compiler: - clangxx cxx_compiler_version: -- '14' +- '15' macos_machine: - arm64-apple-darwin20.0.0 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml similarity index 78% rename from dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml index 6ea00e3bd0d..f75d92e276d 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml @@ -1,3 +1,5 @@ +aws_crt_cpp: +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: @@ -13,9 +15,9 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- '10.2' +- '11.2' cuda_compiler_version_min: -- '10.2' +- '11.2' cxx_compiler: - vs2019 gflags: @@ -23,28 +25,30 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- '2.12' libabseil: - '20230125' libcrc32c: - '1.1' libcurl: -- '7' +- '8' libgrpc: -- '1.52' +- '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -55,7 +59,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -63,6 +67,10 @@ target_platform: thrift_cpp: - 0.18.1 zip_keys: +- - cuda_compiler + - cuda_compiler_version +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml index 183356662c6..6d8fb15b15a 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml @@ -1,3 +1,5 @@ +aws_crt_cpp: +- 0.20.3 aws_sdk_cpp: - 1.10.57 bzip2: @@ -11,11 +13,11 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- nvcc +- None cuda_compiler_version: - None cuda_compiler_version_min: -- '10.2' +- '11.2' cxx_compiler: - vs2019 gflags: @@ -23,28 +25,30 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.8.0 +- '2.12' libabseil: - '20230125' libcrc32c: - '1.1' libcurl: -- '7' +- '8' libgrpc: -- '1.52' +- '1.54' +- '1.56' libprotobuf: - '3.21' +- 4.23.3 lz4_c: - 1.9.3 numpy: - '1.21' - '1.23' -- '1.20' -- '1.20' +- '1.21' +- '1.21' openssl: - '3' orc: -- 1.8.3 +- 1.9.0 pin_run_as_build: python: min_pin: x.x @@ -55,7 +59,7 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2023.02.02 +- 2023.03.02 snappy: - '1' target_platform: @@ -63,6 +67,10 @@ target_platform: thrift_cpp: - 0.18.1 zip_keys: +- - cuda_compiler + - cuda_compiler_version +- - libgrpc + - libprotobuf - - python - numpy zlib: diff --git a/dev/tasks/conda-recipes/arrow-cpp/activate.sh b/dev/tasks/conda-recipes/arrow-cpp/activate.sh index 90210fac0a0..8757612781b 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/activate.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/activate.sh @@ -7,24 +7,70 @@ # doesn't come with a deactivate script, because the symlink # is benign and doesn't need to be deleted. -# where the GDB wrappers get installed -GDB_PREFIX=$CONDA_PREFIX/share/gdb/auto-load +_la_log() { + if [ "${CF_LIBARROW_ACTIVATE_LOGGING:-}" = "1" ]; then + # The following loop is necessary to handle multi-line strings + # like for the output of `ls -al`. + printf '%s\n' "$*" | while IFS= read -r line + do + echo "$CONDA_PREFIX/etc/conda/activate.d/libarrow_activate.sh DEBUG: $line" + done + fi +} + +_la_log "Beginning libarrow activation." -# If the directory is not writable, nothing can be done -if [ ! -w $GDB_PREFIX ]; then - return -fi +# where the GDB wrappers get installed +_la_gdb_prefix="$CONDA_PREFIX/share/gdb/auto-load" -# this needs to be in sync with the respective patch -PLACEHOLDER=replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX +# this needs to be in sync with ARROW_GDB_INSTALL_DIR in build.sh +_la_placeholder="replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX" # the paths here are intentionally stacked, see #935, resp. # https://github.com/apache/arrow/blob/master/docs/source/cpp/gdb.rst#manual-loading -WRAPPER_DIR=$GDB_PREFIX/$CONDA_PREFIX/lib +_la_symlink_dir="$_la_gdb_prefix/$CONDA_PREFIX/lib" +_la_orig_install_dir="$_la_gdb_prefix/$_la_placeholder/lib" -mkdir -p $WRAPPER_DIR -# there's only one lib in that folder, but the libname changes +_la_log " _la_gdb_prefix: $_la_gdb_prefix" +_la_log " _la_placeholder: $_la_placeholder" +_la_log " _la_symlink_dir: $_la_symlink_dir" +_la_log " _la_orig_install_dir: $_la_orig_install_dir" +_la_log " content of that folder:" +_la_log "$(ls -al "$_la_orig_install_dir" | sed 's/^/ /')" + +# there's only one lib in the _la_orig_install_dir folder, but the libname changes # based on the version so use a loop instead of hardcoding it. -for f in $GDB_PREFIX/$PLACEHOLDER/lib/*.py; do - # overwrite, because we don't have deactivation (i.e. symlink remains) - ln -sf $f $WRAPPER_DIR/$(basename $f) +for _la_target in "$_la_orig_install_dir/"*.py; do + if [ ! -e "$_la_target" ]; then + # If the file doesn't exist, skip this iteration of the loop. + # (This happens when no files are found, in which case the + # loop runs with target equal to the pattern itself.) + _la_log 'Folder $_la_orig_install_dir seems to not contain .py files, skipping' + continue + fi + _la_symlink="$_la_symlink_dir/$(basename "$_la_target")" + _la_log " _la_target: $_la_target" + _la_log " _la_symlink: $_la_symlink" + if [ -L "$_la_symlink" ] && [ "$(readlink "$_la_symlink")" = "$_la_target" ]; then + _la_log 'symlink $_la_symlink already exists and points to $_la_target, skipping.' + continue + fi + _la_log 'Creating symlink $_la_symlink pointing to $_la_target' + mkdir -p "$_la_symlink_dir" || true + # this check also creates the symlink; if it fails, we enter the if-branch. + if ! ln -sf "$_la_target" "$_la_symlink"; then + echo -n "${BASH_SOURCE[0]} ERROR: Failed to create symlink from " + echo -n "'$_la_target' to '$_la_symlink'" + echo + continue + fi done + +_la_log "Libarrow activation complete." + +unset _la_gdb_prefix +unset _la_log +unset _la_orig_install_dir +unset _la_placeholder +unset _la_symlink +unset _la_symlink_dir +unset _la_target diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.bat similarity index 89% rename from dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat rename to dev/tasks/conda-recipes/arrow-cpp/build-arrow.bat index 60c81be7411..1268771643d 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.bat @@ -1,16 +1,12 @@ @echo on -mkdir "%SRC_DIR%"\cpp\build -pushd "%SRC_DIR%"\cpp\build +mkdir cpp\build +pushd cpp\build :: Enable CUDA support if "%cuda_compiler_version%"=="None" ( set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=OFF" ) else ( - REM this should move to nvcc-feedstock - set "CUDA_PATH=%CUDA_PATH:\=/%" - set "CUDA_HOME=%CUDA_HOME:\=/%" - set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=ON" ) @@ -18,8 +14,9 @@ if "%cuda_compiler_version%"=="None" ( set "READ_RECIPE_META_YAML_WHY_NOT=OFF" :: for available switches see -:: https://github.com/apache/arrow/blame/apache-arrow-11.0.0/cpp/cmake_modules/DefineOptions.cmake +:: https://github.com/apache/arrow/blame/apache-arrow-12.0.0/cpp/cmake_modules/DefineOptions.cmake cmake -G "Ninja" ^ + -DARROW_ACERO=ON ^ -DARROW_BOOST_USE_SHARED:BOOL=ON ^ -DARROW_BUILD_STATIC:BOOL=OFF ^ -DARROW_BUILD_TESTS:BOOL=OFF ^ @@ -69,3 +66,6 @@ cmake --build . --target install --config Release if %ERRORLEVEL% neq 0 exit 1 popd + +:: clean up between builds (and to save space) +rmdir /s /q cpp\build diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index fb8cbade865..dc588f94738 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -30,27 +30,21 @@ fi # Enable CUDA support if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]] then - if [[ -z "${CUDA_HOME+x}" ]] - then - echo "cuda_compiler_version=${cuda_compiler_version} CUDA_HOME=$CUDA_HOME" - CUDA_GDB_EXECUTABLE=$(which cuda-gdb || exit 0) - if [[ -n "$CUDA_GDB_EXECUTABLE" ]] - then - CUDA_HOME=$(dirname $(dirname $CUDA_GDB_EXECUTABLE)) - else - echo "Cannot determine CUDA_HOME: cuda-gdb not in PATH" - return 1 - fi - fi - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CUDA_HOME}/lib64/stubs" + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib" else EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" fi -if [[ "${target_platform}" == "osx-arm64" ]]; then - EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCLANG_EXECUTABLE=${BUILD_PREFIX}/bin/clang -DLLVM_LINK_EXECUTABLE=${BUILD_PREFIX}/bin/llvm-link" +if [[ "${build_platform}" != "${target_platform}" ]]; then + # point to a usable protoc/grpc_cpp_plugin if we're cross-compiling + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc" + if [[ ! -f ${BUILD_PREFIX}/bin/${CONDA_TOOLCHAIN_HOST}-clang ]]; then + ln -sf ${BUILD_PREFIX}/bin/clang ${BUILD_PREFIX}/bin/${CONDA_TOOLCHAIN_HOST}-clang + fi + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCLANG_EXECUTABLE=${BUILD_PREFIX}/bin/${CONDA_TOOLCHAIN_HOST}-clang" + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DLLVM_LINK_EXECUTABLE=${BUILD_PREFIX}/bin/llvm-link" sed -ie "s;protoc-gen-grpc.*$;protoc-gen-grpc=${BUILD_PREFIX}/bin/grpc_cpp_plugin\";g" ../src/arrow/flight/CMakeLists.txt - sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=14";g' ../cmake_modules/ThirdpartyToolchain.cmake + sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=16";g' ../cmake_modules/ThirdpartyToolchain.cmake fi # disable -fno-plt, which causes problems with GCC on PPC @@ -59,23 +53,19 @@ if [[ "$target_platform" == "linux-ppc64le" ]]; then CXXFLAGS="$(echo $CXXFLAGS | sed 's/-fno-plt //g')" fi -# Limit number of threads used to avoid hardware oversubscription if [[ "${target_platform}" == "linux-aarch64" ]] || [[ "${target_platform}" == "linux-ppc64le" ]]; then - export CMAKE_BUILD_PARALLEL_LEVEL=3 -fi - -# point to a usable protoc if we're running on a different architecture than the target -if [[ "${build_platform}" != "${target_platform}" ]]; then - EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc" + # Limit number of threads used to avoid hardware oversubscription + export CMAKE_BUILD_PARALLEL_LEVEL=3 fi # reusable variable for dependencies we cannot yet unvendor export READ_RECIPE_META_YAML_WHY_NOT=OFF # for available switches see -# https://github.com/apache/arrow/blame/apache-arrow-11.0.0/cpp/cmake_modules/DefineOptions.cmake -# placeholder in ARROW_GDB_INSTALL_DIR must match what's used for replacement in activate.sh +# https://github.com/apache/arrow/blame/apache-arrow-12.0.0/cpp/cmake_modules/DefineOptions.cmake +# placeholder in ARROW_GDB_INSTALL_DIR must match _la_placeholder in activate.sh cmake -GNinja \ + -DARROW_ACERO=ON \ -DARROW_BOOST_USE_SHARED=ON \ -DARROW_BUILD_BENCHMARKS=OFF \ -DARROW_BUILD_STATIC=OFF \ @@ -129,3 +119,6 @@ cmake -GNinja \ cmake --build . --target install --config Release popd + +# clean up between builds (and to save space) +rm -rf cpp/build diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.bat similarity index 54% rename from dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat rename to dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.bat index 084faf74e4a..e3eaa32bcf8 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.bat @@ -2,22 +2,10 @@ pushd "%SRC_DIR%"\python -@rem the symlinks for cmake modules don't work here -@rem NOTE: In contrast to conda-forge, they work here as we clone from git. -@rem del cmake_modules\BuildUtils.cmake -@rem del cmake_modules\SetupCxxFlags.cmake -@rem del cmake_modules\CompilerInfo.cmake -@rem del cmake_modules\FindNumPy.cmake -@rem del cmake_modules\FindPythonLibsNew.cmake -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\BuildUtils.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\SetupCxxFlags.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\CompilerInfo.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindNumPy.cmake" cmake_modules\ -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\ - SET ARROW_HOME=%LIBRARY_PREFIX% SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION% SET PYARROW_BUILD_TYPE=release +SET PYARROW_WITH_ACERO=1 SET PYARROW_WITH_DATASET=1 SET PYARROW_WITH_FLIGHT=1 SET PYARROW_WITH_GANDIVA=1 diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh index 14c67ede632..9c12321a1c1 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -6,6 +6,7 @@ export ARROW_HOME=$PREFIX export PARQUET_HOME=$PREFIX export SETUPTOOLS_SCM_PRETEND_VERSION=$PKG_VERSION export PYARROW_BUILD_TYPE=release +export PYARROW_WITH_ACERO=1 export PYARROW_WITH_DATASET=1 export PYARROW_WITH_FLIGHT=1 export PYARROW_WITH_GANDIVA=1 @@ -37,9 +38,9 @@ if [[ "${target_platform}" == osx-* ]]; then CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" fi -# Limit number of threads used to avoid hardware oversubscription if [[ "${target_platform}" == "linux-aarch64" ]] || [[ "${target_platform}" == "linux-ppc64le" ]]; then - export CMAKE_BUILD_PARALLEL_LEVEL=4 + # Limit number of threads used to avoid hardware oversubscription + export CMAKE_BUILD_PARALLEL_LEVEL=4 fi cd python diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 2f79bbe958c..e61034c3075 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -4,7 +4,7 @@ {% set build_ext_version = ARROW_VERSION %} {% set build_ext = "cuda" if cuda_enabled else "cpu" %} {% set proc_build_number = "0" %} -{% set llvm_version = "14" %} +{% set llvm_version = "15" %} # see https://github.com/apache/arrow/blob/apache-arrow-10.0.1/cpp/CMakeLists.txt#L88-L90 {% set so_version = (version.split(".")[0] | int * 100 + version.split(".")[1] | int) ~ "." ~ version.split(".")[2] ~ ".0" %} @@ -21,11 +21,9 @@ build: # for cuda support, building with one version is enough to be compatible with # all later versions, since arrow is only using libcuda, and not libcudart. skip: true # [cuda_compiler_version not in ("None", cuda_compiler_version_min)] - # temporary: skip CUDA on aarch/ppc until cross-compilation works, see - # https://github.com/conda-forge/conda-forge-ci-setup-feedstock/pull/210 - skip: true # [(aarch64 or ppc64le) and (cuda_compiler_version != "None")] + # arrow promises API- & ABI-compatibility along SemVer, see #1096 run_exports: - - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} + - {{ pin_subpackage("libarrow", max_pin="x") }} outputs: - name: apache-arrow-proc @@ -57,30 +55,38 @@ outputs: - exit 0 - name: libarrow - script: build-arrow.sh # [not win] - script: bld-arrow.bat # [win] + script: build-arrow.sh # [unix] + script: build-arrow.bat # [win] version: {{ version }} build: string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} run_exports: - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} + ignore_run_exports_from: + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] + # arrow only uses headers, apparently + - gflags + # shared lib linked on unix, not on win + - glog # [win] ignore_run_exports: - - cudatoolkit + # we don't need all of brotli's run-exports + - libbrotlicommon track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} missing_dso_whitelist: - - "*/libcuda.so.*" # [linux] - - "*/nvcuda.dll" # [win] + - '*/libcuda.so.*' # [linux] + - '*/nvcuda.dll' # [win] requirements: build: - {{ compiler("c") }} - {{ compiler("cxx") }} - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - - clangdev {{ llvm_version }} # [osx and arm64] - - llvmdev {{ llvm_version }} # [osx and arm64] - - gnuconfig # [osx and arm64] # needs to run protoc & grpc_cpp_plugin - libgrpc # [build_platform != target_platform] - libprotobuf # [build_platform != target_platform] + # needed for gandiva + - clangdev {{ llvm_version }} # [build_platform != target_platform] + - llvmdev {{ llvm_version }} # [build_platform != target_platform] + - gnuconfig # [build_platform != target_platform] - cmake - ninja # necessary for vendored jemalloc @@ -91,12 +97,11 @@ outputs: # https://github.com/apache/arrow/blob/apache-arrow-11.0.0/cpp/cmake_modules/ThirdpartyToolchain.cmake#L46-L75 - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} - - aws-crt-cpp # [unix] + - aws-crt-cpp - aws-sdk-cpp - boost-cpp >=1.70 - brotli - bzip2 - - c-ares # not yet: https://github.com/conda-forge/cpp-opentelemetry-sdk-feedstock/issues/38 # - cpp-opentelemetry-sdk # - proto-opentelemetry-proto =={{ cpp_opentelemetry_sdk }} @@ -106,11 +111,6 @@ outputs: # arrow uses a customized jemalloc, see #944 # - jemalloc - libabseil - # since libgoogle-cloud is static on windows, see - # https://github.com/conda-forge/google-cloud-cpp-feedstock/pull/108, - # its dependencies leak into the build here - - libcrc32c # [win] - - libcurl # [win] - libgrpc - libprotobuf - libutf8proc @@ -127,17 +127,26 @@ outputs: - xsimd - zlib - zstd + - __cuda >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] + # since libgoogle-cloud is static on windows, see + # https://github.com/conda-forge/google-cloud-cpp-feedstock/pull/108, + # its host deps (which aren't yet covered above) leak into the build here + - libcrc32c # [win] + - libcurl # [win] + # same for libgrpc (before 1.55.0, which is coupled with libprotobuf 4.23.x) + - c-ares # [win and libprotobuf == "3.21"] run_constrained: - apache-arrow-proc =*={{ build_ext }} - - cudatoolkit >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] # make sure we don't co-install with old version of old package name - arrow-cpp ={{ version }} + # old parquet lib output, now part of this feedstock + - parquet-cpp <0.0a0 test: commands: {% set headers = [ - "arrow/api.h", "arrow/flight/types.h", "arrow/flight/sql/api.h", - "gandiva/engine.h", "parquet/api/reader.h" + "arrow/api.h", "arrow/acero/api.h", "arrow/flight/types.h", + "arrow/flight/sql/api.h", "gandiva/engine.h", "parquet/api/reader.h" ] %} {% for each_header in headers %} # headers @@ -146,8 +155,8 @@ outputs: {% endfor %} {% set libs = (cuda_compiler_version != "None") * ["arrow_cuda"] + [ - "arrow", "arrow_dataset", "arrow_flight", "arrow_flight_sql", - "arrow_substrait", "gandiva", "parquet" + "arrow", "arrow_acero", "arrow_dataset", "arrow_flight", + "arrow_flight_sql", "arrow_substrait", "gandiva", "parquet" ] %} {% for each_lib in libs %} # shared @@ -189,6 +198,8 @@ outputs: requirements: host: - {{ pin_subpackage('libarrow', exact=True) }} + # avoid wrappers for different builds colliding due to identical hashes + - libprotobuf run: - {{ pin_subpackage('libarrow', exact=True) }} test: @@ -196,21 +207,21 @@ outputs: - exit 0 - name: pyarrow - script: build-pyarrow.sh # [not win] - script: bld-pyarrow.bat # [win] + script: build-pyarrow.sh # [unix] + script: build-pyarrow.bat # [win] version: {{ version }} build: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} - ignore_run_exports: - - cudatoolkit + ignore_run_exports_from: + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} rpaths: - lib/ - {{ SP_DIR }}/pyarrow missing_dso_whitelist: # not actually missing, but installed into SP_DIR, see tests - - "*/arrow_python.dll" # [win] - - "*/arrow_python_flight.dll" # [win] + - '*/arrow_python.dll' # [win] + - '*/arrow_python_flight.dll' # [win] requirements: build: - {{ compiler("c") }} @@ -219,29 +230,28 @@ outputs: - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - - cython # [build_platform != target_platform] + - cython <3 # [build_platform != target_platform] - numpy # [build_platform != target_platform] - cmake - ninja host: - - {{ pin_subpackage('libarrow', exact=True) }} + # we're building for two protobuf versions, cannot pin exactly + # - {{ pin_subpackage('libarrow', exact=True) }} + - libarrow ={{ version }}=*_{{ PKG_BUILDNUM }}_{{ build_ext }} - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} - - cython - - gflags # [unix] + - cython <3 - numpy - python - setuptools - setuptools_scm run: - - {{ pin_subpackage('libarrow', exact=True) }} + # - {{ pin_subpackage('libarrow', exact=True) }} + - libarrow ={{ version }}=*_{{ PKG_BUILDNUM }}_{{ build_ext }} - {{ pin_compatible('numpy') }} - # empty parquet-cpp metapackage, force old versions to be uninstalled - - parquet-cpp 1.5.1.* - python run_constrained: - apache-arrow-proc =*={{ build_ext }} - - cudatoolkit >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] test: files: @@ -288,13 +298,13 @@ outputs: summary: Python libraries for Apache Arrow - name: pyarrow-tests - script: build-pyarrow.sh # [not win] - script: bld-pyarrow.bat # [win] + script: build-pyarrow.sh # [unix] + script: build-pyarrow.bat # [win] version: {{ version }} build: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} - ignore_run_exports: - - cudatoolkit + ignore_run_exports_from: + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} requirements: build: @@ -304,7 +314,7 @@ outputs: - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - - cython # [build_platform != target_platform] + - cython <3 # [build_platform != target_platform] - numpy # [build_platform != target_platform] - cmake - ninja @@ -313,7 +323,7 @@ outputs: - {{ pin_subpackage('pyarrow', exact=True) }} - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} - - cython + - cython <3 - numpy - python - setuptools @@ -323,27 +333,36 @@ outputs: - python run_constrained: - apache-arrow-proc =*={{ build_ext }} - - cudatoolkit >={{ cuda_compiler_version_min }} # [cuda_compiler_version != "None"] + # crossbow CI: reduce to one python version, except on (unemulated) linux, where it's fast enough + {% if linux64 or py == 311 %} + # {% if not (aarch64 or ppc64le) or py in (310, 311) %} + # only run the full test suite for one python version when in emulation (each run takes ~45min); + # there's essentially zero divergence in behaviour across python versions anyway, and otherwise + # CUDA builds for aarch/ppc consistently run out of disk space on azure for some reason test: requires: + # vary protobuf version in test suite (historically, test failures only have a very + # weak dependency on python version, so we don't lose coverage by doing half & half) + - libprotobuf <4 # [py % 2 == 0] # test_cpp_extension_in_python requires a compiler - {{ compiler("cxx") }} # [linux] - - pytest + # temporary pin due to missing fixture + - pytest <7.4.0 - pytest-lazy-fixture - backports.zoneinfo # [py<39] - cffi - cloudpickle - - cython + - cython <3 - fastparquet - fsspec - hypothesis - pandas - scipy - # not all OSes/arches available in conda-forge - - pytorch * # [unix and not ppc64le] - # not yet rebuilt for libabseil 20230125 - # - tensorflow # [unix and x86_64 and py<311] + # these are generally (far) behind on migrating abseil/grpc/protobuf, + # and using them as test dependencies blocks the migrator unnecessarily + # - pytorch + # - tensorflow # we're not building java bindings # - jpype1 # doesn't get picked up correctly @@ -364,6 +383,8 @@ outputs: # skip tests that raise SIGINT and crash the test suite {% set tests_to_skip = tests_to_skip + " or (test_csv and test_cancellation)" %} # [linux] {% set tests_to_skip = tests_to_skip + " or (test_flight and test_interrupt)" %} # [linux] + # tests that may crash the agent due to out-of-bound memory writes or other risky stuff + {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool" %} # [aarch64 or ppc64le] # cannot pass -D_LIBCPP_DISABLE_AVAILABILITY to test suite for our older macos sdk {% set tests_to_skip = tests_to_skip + " or test_cpp_extension_in_python" %} # [osx] # skip tests that make invalid(-for-conda) assumptions about the compilers setup @@ -373,6 +394,7 @@ outputs: {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool_disabled" %} # [aarch64 or ppc64le] {% set tests_to_skip = tests_to_skip + " or test_env_var_io_thread_count" %} # [aarch64 or ppc64le] # vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv + {% set tests_to_skip = tests_to_skip + " or test_extension_to_pandas_storage_type" %} # segfaults on OSX: to investigate ASAP {% set tests_to_skip = tests_to_skip + " or test_flight" %} # [osx] # gandiva tests are segfaulting on ppc @@ -385,10 +407,12 @@ outputs: {% set tests_to_skip = tests_to_skip + " or (test_memory and test_env_var)" %} # [unix] # test is broken; header is in $PREFIX, not $SP_DIR {% set tests_to_skip = tests_to_skip + " or (test_misc and test_get_include)" %} # [unix] + # flaky tests that fail occasionally + {% set tests_to_skip = tests_to_skip + " or test_total_bytes_allocated " %} # [linux] + {% set tests_to_skip = tests_to_skip + " or test_feather_format " %} # [linux] # ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^ - - # crossbow CI: reduce to one python version, except on (unemulated) linux, where it's fast enough - - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [linux64 or (py==310 and build_platform==target_platform)] + - pytest -rfEs -k "not ({{ tests_to_skip }})" + {% endif %} about: home: http://github.com/apache/arrow diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index b9a54647cc5..279ffb48ccd 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -13,6 +13,29 @@ jobs: UPLOAD_PACKAGES: False steps: + - script: | + sudo mkdir -p /opt/empty_dir || true + for d in \ + /opt/ghc \ + /opt/hostedtoolcache \ + /usr/lib/jvm \ + /usr/local/.ghcup \ + /usr/local/lib/android \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift \ + ; do + sudo rsync --stats -a --delete /opt/empty_dir/ $d || true + done + sudo apt-get purge -y -f firefox \ + google-chrome-stable \ + microsoft-edge-stable + sudo apt-get autoremove -y >& /dev/null + sudo apt-get autoclean -y >& /dev/null + sudo docker image prune --all --force + df -h + displayName: Manage disk space + # configure qemu binfmt-misc running. This allows us to run docker containers # embedded qemu-static - script: | diff --git a/dev/tasks/conda-recipes/r-arrow/configure.win b/dev/tasks/conda-recipes/r-arrow/configure.win index fb16a810b8f..0fc96576bde 100755 --- a/dev/tasks/conda-recipes/r-arrow/configure.win +++ b/dev/tasks/conda-recipes/r-arrow/configure.win @@ -2,7 +2,7 @@ set -euxo pipefail -echo "PKG_CPPFLAGS=-DNDEBUG -I\"${LIBRARY_PREFIX}/include\" -I\"${PREFIX}/include\" -DARROW_R_WITH_PARQUET -DARROW_R_WITH_DATASET -DARROW_R_WITH_S3 -DARROW_R_WITH_JSON" > src/Makevars.win +echo "PKG_CPPFLAGS=-DNDEBUG -I\"${LIBRARY_PREFIX}/include\" -I\"${PREFIX}/include\" -DARROW_R_WITH_ACERO -DARROW_R_WITH_PARQUET -DARROW_R_WITH_DATASET -DARROW_R_WITH_S3 -DARROW_R_WITH_JSON" > src/Makevars.win echo "PKG_CXXFLAGS=\$(CXX_VISIBILITY)" >> src/Makevars.win echo 'CXX_STD=CXX17' >> src/Makevars.win -echo "PKG_LIBS=-L\"${LIBRARY_PREFIX}/lib\" -larrow_dataset -lparquet -larrow" >> src/Makevars.win +echo "PKG_LIBS=-L\"${LIBRARY_PREFIX}/lib\" -larrow_dataset -larrow_acero -lparquet -larrow" >> src/Makevars.win diff --git a/dev/tasks/conda-recipes/r-arrow/meta.yaml b/dev/tasks/conda-recipes/r-arrow/meta.yaml index 28ee8eb92c9..e8b834254f4 100644 --- a/dev/tasks/conda-recipes/r-arrow/meta.yaml +++ b/dev/tasks/conda-recipes/r-arrow/meta.yaml @@ -10,8 +10,6 @@ source: path: ../../../../ build: - # 4.1 not usable anymore unless https://github.com/conda-forge/r-base-feedstock/pull/236 gets merged - skip: true # [unix and (r_base == "4.1")] merge_build_host: true # [win] number: 0 rpaths: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 879c2246b41..284946a8cd4 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -246,6 +246,16 @@ tasks: # generated and to be synced regularly from the feedstock. We have no way # yet to generate them inside the arrow repository automatically. + conda-linux-x64-cpu-r41: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_64_cuda_compiler_versionNone + r_config: linux_64_r_base4.1 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + conda-linux-x64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml @@ -272,7 +282,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_cuda_compiler_version10.2 + config: linux_64_cuda_compiler_version11.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda @@ -282,6 +292,16 @@ tasks: ########################### Conda Linux (aarch64) ########################### + conda-linux-aarch64-cpu-r41: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_aarch64_cuda_compiler_versionNone + r_config: linux_aarch64_r_base4.1 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + conda-linux-aarch64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml @@ -304,6 +324,18 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda + conda-linux-aarch64-cuda-py3: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_aarch64_cuda_compiler_version11.2 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda + ########################### Conda Linux (ppc64le) ########################### conda-linux-ppc64le-cpu-py3: @@ -318,8 +350,30 @@ tasks: - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cpu.conda + conda-linux-ppc64le-cuda-py3: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_ppc64le_cuda_compiler_version11.2 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cuda.conda + - pyarrow-{no_rc_version}-py311(h[a-z0-9]+)_0_cuda.conda + ############################## Conda OSX (x64) ############################## + conda-osx-x64-cpu-r41: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_64_ + r_config: osx_64_r_base4.1 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + conda-osx-x64-cpu-r42: ci: azure template: conda-recipes/azure.osx.yml @@ -344,6 +398,16 @@ tasks: ############################# Conda OSX (arm64) ############################# + conda-osx-arm64-cpu-r41: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_arm64_ + r_config: osx_arm64_r_base4.1 + artifacts: + - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda + - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + conda-osx-arm64-cpu-r42: ci: azure template: conda-recipes/azure.osx.yml @@ -396,7 +460,7 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNone + config: win_64_cuda_compiler_version11.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda