diff --git a/.env b/.env index 5345a27149f..2f06cca474b 100644 --- a/.env +++ b/.env @@ -93,12 +93,13 @@ DEVTOOLSET_VERSION=-1 # Please also update the crossbow configuration in order to keep the github # actions cache up to date for the macOS wheels: # https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml -# -# After commit 89295c9 openssl is only available as 3.0.2 which is -# incompatible with aws-sdk-cpp 1.8.3 see -# https://github.com/aws/aws-sdk-cpp/issues/1582 and causes issues on -# manylinux2010 -VCPKG="89295c9" +VCPKG="38bb87c" + +# This must be updated when we update +# ci/docker/python-wheel-windows-vs2017.dockerfile. +# This is a workaround for our CI problem that "archery docker build" doesn't +# use pulled built images in dev/tasks/python-wheels/github.windows.yml. +PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12 # Use conanio/${CONAN} for "docker-compose run --rm conan". See # https://github.com/conan-io/conan-docker-tools#readme for available diff --git a/ci/conan/all/conanfile.py b/ci/conan/all/conanfile.py index f34d6b69791..e0b325bfdaa 100644 --- a/ci/conan/all/conanfile.py +++ b/ci/conan/all/conanfile.py @@ -295,11 +295,7 @@ def requirements(self): if tools.Version(self.version) >= "6.0.0" and \ self.options.get_safe("simd_level") != None or \ self.options.get_safe("runtime_simd_level") != None: - if tools.Version(self.version) >= "8.0.0": - # TODO: Requires xsimd/master - pass - else: - self.requires("xsimd/8.0.3") + self.requires("xsimd/8.1.0") if self.options.with_zlib: self.requires("zlib/1.2.12") if self.options.with_zstd: @@ -581,11 +577,7 @@ def package_info(self): if self.options.with_snappy: self.cpp_info.components["libarrow"].requires.append("snappy::snappy") if self.options.get_safe("simd_level") != None or self.options.get_safe("runtime_simd_level") != None: - if tools.Version(self.version) >= "8.0.0": - # Requires xsimd/master - pass - else: - self.cpp_info.components["libarrow"].requires.append("xsimd::xsimd") + self.cpp_info.components["libarrow"].requires.append("xsimd::xsimd") if self.options.with_zlib: self.cpp_info.components["libarrow"].requires.append("zlib::zlib") if self.options.with_zstd: diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index 7120c56ebb1..dd313f19d70 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -43,6 +43,7 @@ rapidjson re2 snappy thrift-cpp>=0.11.0 +xsimd zlib zstd flatbuffers diff --git a/ci/docker/debian-10-cpp.dockerfile b/ci/docker/debian-10-cpp.dockerfile index 15c85b868ba..aae789164be 100644 --- a/ci/docker/debian-10-cpp.dockerfile +++ b/ci/docker/debian-10-cpp.dockerfile @@ -103,4 +103,5 @@ ENV ARROW_BUILD_TESTS=ON \ ORC_SOURCE=BUNDLED \ PATH=/usr/lib/ccache/:$PATH \ Protobuf_SOURCE=BUNDLED \ + xsimd_SOURCE=BUNDLED \ zstd_SOURCE=BUNDLED diff --git a/ci/docker/debian-11-cpp.dockerfile b/ci/docker/debian-11-cpp.dockerfile index dfccd85e559..790a22dcd80 100644 --- a/ci/docker/debian-11-cpp.dockerfile +++ b/ci/docker/debian-11-cpp.dockerfile @@ -103,4 +103,5 @@ ENV ARROW_BUILD_TESTS=ON \ google_cloud_cpp_storage_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ PATH=/usr/lib/ccache/:$PATH \ - Protobuf_SOURCE=BUNDLED + Protobuf_SOURCE=BUNDLED \ + xsimd_SOURCE=BUNDLED diff --git a/ci/docker/fedora-35-cpp.dockerfile b/ci/docker/fedora-35-cpp.dockerfile index 947c9aba1b7..cbe9ee7539a 100644 --- a/ci/docker/fedora-35-cpp.dockerfile +++ b/ci/docker/fedora-35-cpp.dockerfile @@ -95,6 +95,7 @@ ENV ARROW_BUILD_TESTS=ON \ CXX=g++ \ google_cloud_cpp_storage_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ - PARQUET_BUILD_EXECUTABLES=ON \ PARQUET_BUILD_EXAMPLES=ON \ - PATH=/usr/lib/ccache/:$PATH + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + xsimd_SOURCE=BUNDLED diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile index d17ca0f85be..3a209f6cd2b 100644 --- a/ci/docker/python-wheel-manylinux-201x.dockerfile +++ b/ci/docker/python-wheel-manylinux-201x.dockerfile @@ -81,7 +81,8 @@ RUN vcpkg install \ --x-feature=flight \ --x-feature=gcs \ --x-feature=json \ - --x-feature=parquet + --x-feature=parquet \ + --x-feature=s3 ARG python=3.8 ENV PYTHON_VERSION=${python} diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 026c40b2af7..247f13a15cf 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env +# when you update this file. + # based on mcr.microsoft.com/windows/servercore:ltsc2019 # contains choco and vs2017 preinstalled FROM abrarov/msvc-2017:2.11.0 diff --git a/ci/docker/ubuntu-18.04-cpp.dockerfile b/ci/docker/ubuntu-18.04-cpp.dockerfile index ed038f4c4c5..16490845bd0 100644 --- a/ci/docker/ubuntu-18.04-cpp.dockerfile +++ b/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -129,4 +129,5 @@ ENV ARROW_BUILD_TESTS=ON \ PATH=/usr/lib/ccache/:$PATH \ Thrift_SOURCE=BUNDLED \ utf8proc_SOURCE=BUNDLED \ + xsimd_SOURCE=BUNDLED \ zstd_SOURCE=BUNDLED diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 0eade393cd4..ae15835520c 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -150,11 +150,12 @@ ENV ARROW_BUILD_TESTS=ON \ ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \ AWSSDK_SOURCE=BUNDLED \ google_cloud_cpp_storage_SOURCE=BUNDLED \ - GTest_SOURCE=BUNDLED \ gRPC_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ ORC_SOURCE=BUNDLED \ PARQUET_BUILD_EXAMPLES=ON \ PARQUET_BUILD_EXECUTABLES=ON \ - PATH=/usr/lib/ccache/:$PATH \ Protobuf_SOURCE=BUNDLED \ - PYTHON=python3 + PATH=/usr/lib/ccache/:$PATH \ + PYTHON=python3 \ + xsimd_SOURCE=BUNDLED diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index 1398dcd636a..e7d2842dfce 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -181,6 +181,7 @@ ENV ARROW_BUILD_TESTS=ON \ ORC_SOURCE=BUNDLED \ PARQUET_BUILD_EXAMPLES=ON \ PARQUET_BUILD_EXECUTABLES=ON \ - Protobuf_SOURCE=BUNDLED \ PATH=/usr/lib/ccache/:$PATH \ - PYTHON=python3 + Protobuf_SOURCE=BUNDLED \ + PYTHON=python3 \ + xsimd_SOURCE=BUNDLED diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index d88d95b6774..8e823b05396 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -161,6 +161,7 @@ cmake \ -DThrift_SOURCE=${Thrift_SOURCE:-} \ -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \ -Dzstd_SOURCE=${zstd_SOURCE:-} \ + -Dxsimd_SOURCE=${xsimd_SOURCE:-} \ -G "${CMAKE_GENERATOR:-Ninja}" \ ${CMAKE_ARGS} \ ${source_dir} diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index 7fa43a3eaa2..a6e763b6523 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -123,7 +123,6 @@ cmake \ -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \ -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \ -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \ - -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh index 6cfd34d851f..af17606199e 100755 --- a/ci/scripts/python_wheel_manylinux_build.sh +++ b/ci/scripts/python_wheel_manylinux_build.sh @@ -117,7 +117,6 @@ cmake \ -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \ -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \ -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \ - -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \ diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index 72072bf886b..fb776185a5f 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -47,6 +47,9 @@ set ARROW_WITH_LZ4=ON set ARROW_WITH_SNAPPY=ON set ARROW_WITH_ZLIB=ON set ARROW_WITH_ZSTD=ON +@rem Workaround for https://github.com/aws/aws-sdk-cpp/issues/1809 . +@rem Use (old) bundled AWS SDK C++ instead of (newer) AWS SDK C++. +set AWSSDK_SOURCE=BUNDLED set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 set VCPKG_ROOT=C:\vcpkg @@ -81,7 +84,7 @@ cmake ^ -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^ -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^ -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^ - -DAWSSDK_SOURCE=BUNDLED ^ + -DAWSSDK_SOURCE=%AWSSDK_SOURCE% ^ -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ -DCMAKE_CXX_COMPILER=clcache ^ -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^ diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch index 83988cb4053..b2eed47466d 100644 --- a/ci/vcpkg/ports.patch +++ b/ci/vcpkg/ports.patch @@ -71,44 +71,6 @@ index 1289eed36a..b010a69f13 100644 ) vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS -diff --git a/ports/aws-sdk-cpp/fix-find-crypto.patch b/ports/aws-sdk-cpp/fix-find-crypto.patch -new file mode 100644 -index 0000000000..aafc631e93 ---- /dev/null -+++ b/ports/aws-sdk-cpp/fix-find-crypto.patch -@@ -0,0 +1,20 @@ -+diff --git a/cmake/setup_cmake_find_module.cmake b/cmake/setup_cmake_find_module.cmake -+index f5f147a0f..4561b8c39 100644 -+--- a/cmake/setup_cmake_find_module.cmake -++++ b/cmake/setup_cmake_find_module.cmake -+@@ -30,15 +30,6 @@ file(WRITE -+ "set(AWSSDK_INSTALL_BINDIR ${BINARY_DIRECTORY})\n" -+ "set(AWSSDK_INSTALL_INCLUDEDIR ${INCLUDE_DIRECTORY})\n" -+ "set(AWSSDK_INSTALL_ARCHIVEDIR ${ARCHIVE_DIRECTORY})\n" -+- "if (NOT LibCrypto_INCLUDE_DIR)\n" -+- " set(LibCrypto_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR} CACHE INTERNAL \"The OpenSSL include directory\")\n" -+- "endif()\n" -+- "if (NOT LibCrypto_STATIC_LIBRARY)\n" -+- " set(LibCrypto_STATIC_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto static library\")\n" -+- "endif()\n" -+- "if (NOT LibCrypto_SHARED_LIBRARY)\n" -+- " set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n" -+- "endif()\n" -+ ) -+ -+ if (NOT SIMPLE_INSTALL) -diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake -index b520a17ae3..ea9e770cf5 100644 ---- a/ports/aws-sdk-cpp/portfile.cmake -+++ b/ports/aws-sdk-cpp/portfile.cmake -@@ -8,6 +8,7 @@ vcpkg_from_github( - PATCHES - patch-relocatable-rpath.patch - fix-aws-root.patch -+ fix-find-crypto.patch - ) - - string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT) diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake index f81d0c491d..e5ea9cef57 100644 --- a/ports/curl/portfile.cmake diff --git a/ci/vcpkg/vcpkg.json b/ci/vcpkg/vcpkg.json index b62850631c4..d9d074e99b0 100644 --- a/ci/vcpkg/vcpkg.json +++ b/ci/vcpkg/vcpkg.json @@ -25,6 +25,7 @@ "re2", "snappy", "utf8proc", + "xsimd", "zlib", "zstd", { diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 381b136e8db..6685ffb481b 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2204,16 +2204,20 @@ if((NOT ARROW_SIMD_LEVEL STREQUAL "NONE") OR (NOT ARROW_RUNTIME_SIMD_LEVEL STREQ else() set(ARROW_USE_XSIMD FALSE) endif() + if(ARROW_USE_XSIMD) - set(xsimd_SOURCE "BUNDLED") - resolve_dependency(xsimd) + resolve_dependency(xsimd REQUIRED_VERSION "8.1.0") - add_library(xsimd INTERFACE IMPORTED) - if(CMAKE_VERSION VERSION_LESS 3.11) - set_target_properties(xsimd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES - "${XSIMD_INCLUDE_DIR}") + if(xsimd_SOURCE STREQUAL "BUNDLED") + add_library(xsimd INTERFACE IMPORTED) + if(CMAKE_VERSION VERSION_LESS 3.11) + set_target_properties(xsimd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${XSIMD_INCLUDE_DIR}") + else() + target_include_directories(xsimd INTERFACE "${XSIMD_INCLUDE_DIR}") + endif() else() - target_include_directories(xsimd INTERFACE "${XSIMD_INCLUDE_DIR}") + message(STATUS "xsimd found. Headers: ${xsimd_INCLUDE_DIRS}") endif() endif() @@ -4437,6 +4441,14 @@ macro(build_awssdk) -DENABLE_UNITY_BUILD=ON "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}" "-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}") + if(NOT MSVC) + list(APPEND + AWSSDK_COMMON_CMAKE_ARGS + # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1582 + "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -Wno-error=deprecated-declarations" + "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS} -Wno-error=deprecated-declarations" + ) + endif() # provide hint for AWS SDK to link with the already located openssl get_filename_component(OPENSSL_ROOT_HINT "${OPENSSL_INCLUDE_DIR}" DIRECTORY) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 69dc4486cd6..589f7402ba7 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -93,8 +93,8 @@ ARROW_UCX_BUILD_VERSION=1.12.1 ARROW_UCX_BUILD_SHA256_CHECKSUM=9bef31aed0e28bf1973d28d74d9ac4f8926c43ca3b7010bd22a084e164e31b71 ARROW_UTF8PROC_BUILD_VERSION=v2.7.0 ARROW_UTF8PROC_BUILD_SHA256_CHECKSUM=4bb121e297293c0fd55f08f83afab6d35d48f0af4ecc07523ad8ec99aa2b12a1 -ARROW_XSIMD_BUILD_VERSION=7d1778c3b38d63db7cec7145d939f40bc5d859d1 -ARROW_XSIMD_BUILD_SHA256_CHECKSUM=d861f4f3034550cdc62ad93fd60e8b0f2413a6ea49081c8698922b7b4f043ec6 +ARROW_XSIMD_BUILD_VERSION=8.1.0 +ARROW_XSIMD_BUILD_SHA256_CHECKSUM=d52551360d37709675237d2a0418e28f70995b5b7cdad7c674626bcfbbf48328 ARROW_ZLIB_BUILD_VERSION=1.2.12 ARROW_ZLIB_BUILD_SHA256_CHECKSUM=91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9 ARROW_ZSTD_BUILD_VERSION=v1.5.1 diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json index 8e7fe75f5ad..8e527adec39 100644 --- a/cpp/vcpkg.json +++ b/cpp/vcpkg.json @@ -47,6 +47,7 @@ "sqlite3", "thrift", "utf8proc", + "xsimd", "zlib", "zstd" ], diff --git a/dev/archery/archery/docker/cli.py b/dev/archery/archery/docker/cli.py index c6b4a6473b8..bbdd2261db6 100644 --- a/dev/archery/archery/docker/cli.py +++ b/dev/archery/archery/docker/cli.py @@ -79,6 +79,36 @@ def check_config(obj): # during the configuration loading +@docker.command('pull') +@click.argument('image') +@click.option('--using-docker-cli', default=False, is_flag=True, + envvar='ARCHERY_USE_DOCKER_CLI', + help="Use docker CLI directly for pulling instead of calling " + "docker-compose. This may help to reuse cached layers.") +@click.option('--pull-leaf/--no-leaf', default=True, + help="Whether to pull leaf images too.") +@click.option('--ignore-pull-failures/--no-ignore-pull-failures', default=True, + help="Whether to ignore pull failures.") +@click.pass_obj +def docker_pull(obj, image, *, using_docker_cli, pull_leaf, + ignore_pull_failures): + """ + Execute docker-compose pull. + """ + compose = obj['compose'] + + try: + compose.pull(image, pull_leaf=pull_leaf, using_docker=using_docker_cli, + ignore_pull_failures=ignore_pull_failures) + except UndefinedImage as e: + raise click.ClickException( + "There is no service/image defined in docker-compose.yml with " + "name: {}".format(str(e)) + ) + except RuntimeError as e: + raise click.ClickException(str(e)) + + @docker.command('build') @click.argument('image') @click.option('--force-pull/--no-pull', default=True, diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py index dfdf3f4778a..4d6da095b80 100644 --- a/dev/archery/archery/docker/core.py +++ b/dev/archery/archery/docker/core.py @@ -219,7 +219,8 @@ def _execute_docker(self, *args, **kwargs): ) ) - def pull(self, service_name, pull_leaf=True, using_docker=False): + def pull(self, service_name, pull_leaf=True, using_docker=False, + ignore_pull_failures=True): def _pull(service): args = ['pull'] if service['image'] in self.pull_memory: @@ -229,10 +230,14 @@ def _pull(service): try: self._execute_docker(*args, service['image']) except Exception as e: - # better --ignore-pull-failures handling - print(e) + if ignore_pull_failures: + # better --ignore-pull-failures handling + print(e) + else: + raise else: - args.append('--ignore-pull-failures') + if ignore_pull_failures: + args.append('--ignore-pull-failures') self._execute_compose(*args, service['name']) self.pull_memory.add(service['image']) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 72fbda9c96d..dcf1afe4a1a 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -90,6 +90,7 @@ outputs: - re2 - snappy - thrift-cpp + - xsimd - zlib - zstd run: diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 420b52c088f..505a17cebee 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -57,6 +57,7 @@ jobs: - script: | source activate base + set -e set +x if [[ "${CONFIG}" == osx_arm* ]]; then EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index b8e5669fa2e..4e7fc4cf352 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -38,7 +38,7 @@ on: {%- macro github_login_dockerhub() -%} - name: Login to Dockerhub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: {{ '${{ secrets.DOCKERHUB_USER }}' }} password: {{ '${{ secrets.DOCKERHUB_TOKEN }}' }} @@ -46,8 +46,11 @@ on: {%- macro github_login_ghcr() -%} - name: Login to GitHub Container Registry - shell: bash - run: docker login ghcr.io -u {{ '${{ github.repository_owner }}' }} -p {{ '${{ secrets.CROSSBOW_GHCR_TOKEN }}' }} + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: {{ '${{ github.actor }}' }} + password: {{ '${{ secrets.GITHUB_TOKEN }}' }} {% endmacro %} {%- macro github_install_archery() -%} @@ -292,4 +295,4 @@ on: # getwd() is necessary as this macro is used within jobs using a docker container tools::write_PACKAGES(file.path(getwd(), "/repo/src/contrib", fsep = "/"), type = "source", verbose = TRUE) - run: ls -R repo -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml index c18e080ac21..b647c3c7f49 100644 --- a/dev/tasks/python-wheels/github.osx.amd64.yml +++ b/dev/tasks/python-wheels/github.osx.amd64.yml @@ -74,6 +74,11 @@ jobs: - name: Install Packages run: | + if [ "${ARROW_S3}" == "ON" ]; then + x_feature_s3="--x-feature=s3" + else + x_feature_s3= + fi vcpkg install \ --clean-after-build \ --x-install-root=${VCPKG_ROOT}/installed \ @@ -81,7 +86,8 @@ jobs: --x-feature=flight \ --x-feature=gcs \ --x-feature=json \ - --x-feature=parquet + --x-feature=parquet \ + ${x_feature_s3} - name: Install Python {{ python_version }} shell: bash diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml index 7198da7de47..f1ad9a7927b 100644 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -75,7 +75,8 @@ jobs: --x-feature=flight \ --x-feature=gcs \ --x-feature=json \ - --x-feature=parquet + --x-feature=parquet \ + --x-feature=s3 - name: Build ARM64 Wheel env: @@ -99,7 +100,8 @@ jobs: --x-feature=flight \ --x-feature=gcs \ --x-feature=json \ - --x-feature=parquet + --x-feature=parquet \ + --x-feature=s3 - name: Build AMD64 Wheel env: diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index ee4ebb96fb9..6694e9feca6 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -43,11 +43,34 @@ jobs: - name: Build wheel shell: cmd - run: archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017 + run: | + cd arrow + @rem We want to use only + @rem archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017 + @rem but it doesn't use pulled caches. + @rem It always build an image from scratch. + @rem We can remove this workaround once we find a way to use + @rem pulled caches when build an image. + echo on + archery docker pull --no-ignore-pull-failures python-wheel-windows-vs2017 + if errorlevel 1 ( + archery docker build --no-pull python-wheel-windows-vs2017 || exit /B 1 + ) + archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017 - name: Test wheel shell: cmd - run: archery docker run python-wheel-windows-test + run: | + cd arrow + archery docker run python-wheel-windows-test {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }} {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }} + + {% if arrow.branch == 'master' %} + - name: Push Docker Image + shell: cmd + run: | + cd arrow + archery docker push python-wheel-windows-vs2017 + {% endif %} diff --git a/docker-compose.yml b/docker-compose.yml index db7f4f1dfd7..7a8ca192f9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -945,17 +945,7 @@ services: command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow python-wheel-windows-vs2017: - # The windows images must be built locally and pushed to a remote registry: - # export REPO=ghcr.io/ursacomputing/arrow - # PYTHON=3.7 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017 - # PYTHON=3.8 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017 - # PYTHON=3.9 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017 - # PYTHON=3.10 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017 - # PYTHON=3.7 archery docker push python-wheel-windows-vs2017 - # PYTHON=3.8 archery docker push python-wheel-windows-vs2017 - # PYTHON=3.9 archery docker push python-wheel-windows-vs2017 - # PYTHON=3.10 archery docker push python-wheel-windows-vs2017 - image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG} + image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} build: args: vcpkg: ${VCPKG} @@ -965,8 +955,8 @@ services: # This should make the pushed images reusable, but the image gets rebuilt. # Uncomment if no local cache is available. # cache_from: - # - mcr.microsoft.com/windows/servercore:ltsc2019 - # - ${REPO}:wheel-windows-vs2017 + # - abrarov/msvc-2017:2.11.0 + # - ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION} volumes: - "${DOCKER_VOLUME_PREFIX}python-wheel-windows-clcache:C:/clcache" - type: bind