Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
016d4f8
Initial changes
AlenkaF Sep 22, 2025
55516c7
Wait for conda-forge support
AlenkaF Sep 22, 2025
0da81ee
Fix musllinux issue
AlenkaF Sep 22, 2025
700896d
Update test due to getrefcount changes
AlenkaF Sep 23, 2025
522bd9c
Remove 3.14t for now
AlenkaF Sep 23, 2025
20770df
Get the rc image tag from docker hub, use sysconfig.get_config_var(Py…
AlenkaF Sep 25, 2025
d46b035
Use different image tag
AlenkaF Sep 26, 2025
767e2c4
Revert PYTHON_ABI_TAG removal
AlenkaF Sep 26, 2025
4e6706a
Fix macos Python 3.14 install
AlenkaF Oct 3, 2025
c3f4e0c
Add missing fi
AlenkaF Oct 3, 2025
9026095
Add 3.14 for test-conda-python-*
AlenkaF Oct 3, 2025
1853f44
Initial add of free-threaded 3.14 wheels
AlenkaF Oct 7, 2025
51fe247
Revert changes in detecting free-threaded builds as the sysconfig che…
AlenkaF Oct 7, 2025
add8578
Update osx yml
AlenkaF Oct 7, 2025
7f84d55
Try a fix for macos failure
AlenkaF Oct 7, 2025
50d1c5e
Try a fix for musllinux failure
AlenkaF Oct 7, 2025
d0b86b9
Fix linter
AlenkaF Oct 7, 2025
dc5fea0
Fix linter again
AlenkaF Oct 7, 2025
7706a67
Add back hadolint ignore=SC1072
AlenkaF Oct 7, 2025
7ed3b57
Try a fix for free-threaded Windows wheel
AlenkaF Oct 8, 2025
1db44e4
Use official Python version 3.14
AlenkaF Oct 9, 2025
ce1516d
Try removing protobuf symbols export from symbols.map
AlenkaF Oct 9, 2025
a48f067
Fix osx 3.13t
AlenkaF Oct 9, 2025
41d4708
Revert changes to symbols.map
AlenkaF Oct 9, 2025
ea1278b
Another fix for osx wheels
AlenkaF Oct 9, 2025
1030a08
Use quay.io image tag that works for musllinux wheels
AlenkaF Oct 9, 2025
077aa13
Try adding CMAKE_POLICY_VERSION_MINIMUM flag and remove nightly Cyhto…
AlenkaF Oct 10, 2025
6f5f033
Remove unintended commit of ps file
AlenkaF Oct 10, 2025
1c5e76e
Try extra rmdir for win build
AlenkaF Oct 10, 2025
9bab8a0
Add some logging and try building Win wheels without Flight for test …
AlenkaF Oct 10, 2025
e4ad2a3
Revert "Add some logging and try building Win wheels without Flight f…
AlenkaF Oct 10, 2025
abf9e6e
Revert "Try extra rmdir for win build"
AlenkaF Oct 10, 2025
df5f60a
Update Win image revision tag
AlenkaF Oct 13, 2025
a92a23f
Remove COPY of the req file in win dockerfile
AlenkaF Oct 13, 2025
ed83b6e
Add free-threaded project classifier (Beta)
AlenkaF Oct 13, 2025
e59f093
Fix failing linter
AlenkaF Oct 13, 2025
26a7f72
Revert COPY changes
AlenkaF Oct 13, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1" # 2025.09.17 Release
# ci/docker/python-*-windows-*.dockerfile or the vcpkg config.
# This is a workaround for our CI problem that "archery docker build" doesn't
# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-09-04
PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-09-04
PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-10-13
PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-10-13

# Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan".
# See https://github.com/conan-io/conan-docker-tools#readme and
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,20 @@
ARG base
FROM ${base}

ARG python_version=3.13

ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update -y -q && \
apt install -y -q --no-install-recommends software-properties-common gpg-agent && \
add-apt-repository -y ppa:deadsnakes/ppa && \
apt-get update -y -q && \
apt install -y -q --no-install-recommends python3.13-dev python3.13-nogil python3.13-venv && \
apt install -y -q --no-install-recommends python${python_version}-dev python${python_version}-nogil python${python_version}-venv && \
apt-get clean && \
rm -rf /var/lib/apt/lists*

ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
RUN python${python_version}t -m venv ${ARROW_PYTHON_VENV}

ENV PYTHON_GIL 0
ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
ARG base
FROM ${base}

ARG python_version=3.13

ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update -y -q && \
Expand All @@ -27,14 +29,14 @@ RUN apt-get update -y -q && \
apt install -y -q --no-install-recommends \
build-essential \
libffi-dev \
python3.13-dev \
python3.13-nogil \
python3.13-venv && \
python${python_version}-dev \
python${python_version}-nogil \
python${python_version}-venv && \
apt-get clean && \
rm -rf /var/lib/apt/lists*

ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
RUN python${python_version}t -m venv ${ARROW_PYTHON_VENV}

ENV PYTHON_GIL 0
ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
ARG base
FROM ${base}

ARG python_version=3.13
ARG python_patch_version=3.13.7

RUN apk add --no-cache \
bash \
build-base \
Expand All @@ -34,19 +37,19 @@ RUN apk add --no-cache \
tzdata \
zlib-dev

# Install Python3.13.2 without GIL
RUN wget https://github.com/python/cpython/archive/refs/tags/v3.13.2.tar.gz && \
tar -xzf v3.13.2.tar.gz && \
rm v3.13.2.tar.gz && \
cd cpython-3.13.2/ && \
# Install Python without GIL
RUN wget https://github.com/python/cpython/archive/refs/tags/v${python_patch_version}.tar.gz && \
tar -xzf v${python_patch_version}.tar.gz && \
rm v${python_patch_version}.tar.gz && \
cd cpython-${python_patch_version}/ && \
./configure --disable-gil --with-ensurepip && \
make -j && \
make install && \
cd ../ && \
rm -rf cpython-3.13.2/
rm -rf cpython-${python_patch_version}/

ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
RUN python${python_version}t -m venv ${ARROW_PYTHON_VENV}

ENV PYTHON_GIL 0
ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
ARG base
FROM ${base}

ARG python_version=3.13
ARG python_patch_version=3.13.7

RUN apk add --no-cache \
bash \
build-base \
Expand All @@ -34,19 +37,19 @@ RUN apk add --no-cache \
tzdata \
zlib-dev

# Install Python3.13.2 without GIL
RUN wget https://github.com/python/cpython/archive/refs/tags/v3.13.2.tar.gz && \
tar -xzf v3.13.2.tar.gz && \
rm v3.13.2.tar.gz && \
cd cpython-3.13.2/ && \
# Install Python without GIL
RUN wget https://github.com/python/cpython/archive/refs/tags/v${python_patch_version}.tar.gz && \
tar -xzf v${python_patch_version}.tar.gz && \
rm v${python_patch_version}.tar.gz && \
cd cpython-${python_patch_version}/ && \
./configure --disable-gil --with-ensurepip && \
make -j && \
make install && \
cd ../ && \
rm -rf cpython-3.13.2/
rm -rf cpython-${python_patch_version}/

ENV ARROW_PYTHON_VENV /arrow-dev
RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
RUN python${python_version}t -m venv ${ARROW_PYTHON_VENV}

ENV PYTHON_GIL 0
ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,24 +26,34 @@ FROM ${base}

ARG python=3.13

# hadolint ignore=SC1072
RUN (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1") & \
(if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0")

SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
RUN $filename = 'python-3.13.1-amd64.exe'; \
$url = 'https://www.python.org/ftp/python/3.13.1/' + $filename; \
RUN $version = $env:PYTHON_VERSION; \
$filename = 'python-' + $version + '-amd64.exe'; \
$url = 'https://www.python.org/ftp/python/' + $version + '/' + $filename; \
Invoke-WebRequest -Uri $url -OutFile $filename; \
Start-Process -FilePath $filename -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait

ENV PYTHON_CMD="py -${python}t"

SHELL ["cmd", "/S", "/C"]
RUN %PYTHON_CMD% -m pip install -U pip setuptools

COPY python/requirements-wheel-test-3.13t.txt C:/arrow/python/
# Cython and Pandas wheels for 3.13 free-threaded are not released yet
RUN %PYTHON_CMD% -m pip install -U pip setuptools & \
if "%python%"=="3.13" ( \
setx REQUIREMENTS_FILE "requirements-wheel-test-3.13t.txt" \
) else ( \
setx REQUIREMENTS_FILE "requirements-wheel-test.txt" \
)

COPY python/requirements-wheel-test-3.13t.txt python/requirements-wheel-test.txt C:/arrow/python/
# Cython and Pandas wheels for free-threaded are not released yet
RUN %PYTHON_CMD% -m pip install \
--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
--pre \
--prefer-binary \
-r C:/arrow/python/requirements-wheel-test-3.13t.txt
-r C:/arrow/python/%REQUIREMENTS_FILE%

ENV PYTHON="${python}t"
ENV PYTHON_GIL=0
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,13 @@ FROM ${base}

ARG python=3.13

RUN (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1") & \
(if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0")

SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
RUN $filename = 'python-3.13.1-amd64.exe'; \
$url = 'https://www.python.org/ftp/python/3.13.1/' + $filename; \
RUN $version = $env:PYTHON_VERSION; \
$filename = 'python-' + $version + '-amd64.exe'; \
$url = 'https://www.python.org/ftp/python/' + $version + '/' + $filename; \
Invoke-WebRequest -Uri $url -OutFile $filename; \
Start-Process -FilePath $filename -ArgumentList '/quiet', 'Include_freethreaded=1' -Wait

Expand Down
6 changes: 1 addition & 5 deletions ci/docker/python-wheel-musllinux.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ RUN --mount=type=secret,id=github_repository_owner \
export GITHUB_REPOSITORY_OWNER=$(cat /run/secrets/github_repository_owner); \
export GITHUB_TOKEN=$(cat /run/secrets/github_token); \
export VCPKG_BINARY_SOURCES=$(cat /run/secrets/vcpkg_binary_sources); \
export CMAKE_POLICY_VERSION_MINIMUM=3.5; \
arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
vcpkg install \
--clean-after-build \
Expand Down Expand Up @@ -110,10 +111,5 @@ RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-${PYTHON_ABI_TAG
SHELL ["/bin/bash", "-i", "-c", "-l"]
ENTRYPOINT ["/bin/bash", "-i", "-c", "-l"]

# Remove once there are released Cython wheels for 3.13 free-threaded available
RUN if [ "${python_abi_tag}" = "cp313t" ]; then \
pip install cython --pre --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" --prefer-binary ; \
fi

COPY python/requirements-wheel-build.txt /arrow/python/
RUN pip install -r /arrow/python/requirements-wheel-build.txt
5 changes: 3 additions & 2 deletions ci/docker/python-wheel-windows-test-vs2022.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ FROM ${base}
ARG python=3.10
RUN (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PYTHON_CMD "py -3.10") & \
(if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PYTHON_CMD "py -3.11") & \
(if "%python%"=="3.12" setx PYTHON_VERSION "3.12.8" && setx PYTHON_CMD "py -3.12") & \
(if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1" && setx PYTHON_CMD "py -3.13")
(if "%python%"=="3.12" setx PYTHON_VERSION "3.12.10" && setx PYTHON_CMD "py -3.12") & \
(if "%python%"=="3.13" setx PYTHON_VERSION "3.13.7" && setx PYTHON_CMD "py -3.13") & \
(if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0" && setx PYTHON_CMD "py -3.14")

# hadolint ignore=DL3059
RUN choco install -r -y --pre --no-progress --force python --version=%PYTHON_VERSION%
Expand Down
5 changes: 3 additions & 2 deletions ci/docker/python-wheel-windows-vs2022.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ FROM ${base}
ARG python=3.10
RUN (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PYTHON_CMD "py -3.10") & \
(if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PYTHON_CMD "py -3.11") & \
(if "%python%"=="3.12" setx PYTHON_VERSION "3.12.8" && setx PYTHON_CMD "py -3.12") & \
(if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1" && setx PYTHON_CMD "py -3.13")
(if "%python%"=="3.12" setx PYTHON_VERSION "3.12.10" && setx PYTHON_CMD "py -3.12") & \
(if "%python%"=="3.13" setx PYTHON_VERSION "3.13.7" && setx PYTHON_CMD "py -3.13") & \
(if "%python%"=="3.14" setx PYTHON_VERSION "3.14.0" && setx PYTHON_CMD "py -3.14")

RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION%
RUN %PYTHON_CMD% -m pip install -U pip setuptools
Expand Down
19 changes: 12 additions & 7 deletions ci/scripts/install_python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,11 @@ platforms=([windows]=Windows
declare -A versions
versions=([3.10]=3.10.11
[3.11]=3.11.9
[3.12]=3.12.9
[3.13]=3.13.2
[3.13t]=3.13.2)
[3.12]=3.12.10
[3.13]=3.13.7
[3.13t]=3.13.7
[3.14]=3.14.0
[3.14t]=3.14.0)

if [ "$#" -ne 2 ]; then
echo "Usage: $0 <platform> <version>"
Expand All @@ -50,7 +52,9 @@ if [ "$platform" = "macOS" ]; then
wget "https://www.python.org/ftp/python/${full_version}/${fname}"

echo "Installing Python..."
if [[ $2 == "3.13t" ]]; then
if [[ $2 == "3.13t" ]] || [[ $2 == "3.14t" ]]; then
# Extract the base version without 't' suffix
base_version="${version%t}"
# See https://github.com/python/cpython/issues/120098#issuecomment-2151122033 for more info on this.
cat > ./choicechanges.plist <<EOF
<?xml version="1.0" encoding="UTF-8"?>
Expand All @@ -63,7 +67,7 @@ if [ "$platform" = "macOS" ]; then
<key>choiceAttribute</key>
<string>selected</string>
<key>choiceIdentifier</key>
<string>org.python.Python.PythonTFramework-3.13</string>
<string>org.python.Python.PythonTFramework-${base_version}</string>
</dict>
</array>
</plist>
Expand All @@ -76,8 +80,9 @@ EOF
rm "$fname"

python="/Library/Frameworks/Python.framework/Versions/${version}/bin/python${version}"
if [[ $2 == "3.13t" ]]; then
python="/Library/Frameworks/PythonT.framework/Versions/3.13/bin/python3.13t"
if [[ $2 == "3.13t" ]] || [[ $2 == "3.14t" ]]; then
base_version="${version%t}"
python="/Library/Frameworks/PythonT.framework/Versions/${base_version}/bin/python${base_version}t"
fi

echo "Installing Pip..."
Expand Down
8 changes: 4 additions & 4 deletions ci/scripts/python_wheel_unix_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ source_dir=${1}
: "${ARROW_AZURE:=ON}"
: "${ARROW_FLIGHT:=ON}"
: "${ARROW_GCS:=ON}"
: "${CHECK_IMPORTS:=ON}"
: "${ARROW_S3:=ON}"
: "${ARROW_SUBSTRAIT:=ON}"
: "${CHECK_IMPORTS:=ON}"
: "${CHECK_WHEEL_CONTENT:=ON}"
: "${CHECK_UNITTESTS:=ON}"
: "${INSTALL_PYARROW:=ON}"
Expand Down Expand Up @@ -106,10 +106,10 @@ is_free_threaded() {

if [ "${CHECK_UNITTESTS}" == "ON" ]; then
# Install testing dependencies
if [ "$(is_free_threaded)" = "ON" ]; then
echo "Free-threaded Python build detected"
if [ "$(is_free_threaded)" = "ON" ] && [[ "${PYTHON:-}" == *"3.13"* ]]; then
echo "Free-threaded Python 3.13 build detected"
python -m pip install -U -r "${source_dir}/python/requirements-wheel-test-3.13t.txt"
elif [ "$(is_free_threaded)" = "OFF" ]; then
else
echo "Regular Python build detected"
python -m pip install -U -r "${source_dir}/python/requirements-wheel-test.txt"
fi
Expand Down
6 changes: 3 additions & 3 deletions dev/release/verify-release-candidate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -852,7 +852,7 @@ test_linux_wheels() {
local arch="x86_64"
fi

local python_versions="${TEST_PYTHON_VERSIONS:-3.10 3.11 3.12 3.13}"
local python_versions="${TEST_PYTHON_VERSIONS:-3.10 3.11 3.12 3.13 3.14}"
local platform_tags="${TEST_WHEEL_PLATFORM_TAGS:-manylinux_2_28_${arch}}"

if [ "${SOURCE_KIND}" != "local" ]; then
Expand Down Expand Up @@ -891,11 +891,11 @@ test_macos_wheels() {

# apple silicon processor
if [ "$(uname -m)" = "arm64" ]; then
local python_versions="3.10 3.11 3.12 3.13"
local python_versions="3.10 3.11 3.12 3.13 3.14"
local platform_tags="macosx_12_0_arm64"
local check_flight=OFF
else
local python_versions="3.10 3.11 3.12 3.13"
local python_versions="3.10 3.11 3.12 3.13 3.14"
local platform_tags="macosx_12_0_x86_64"
fi

Expand Down
11 changes: 10 additions & 1 deletion dev/tasks/python-wheels/github.linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,25 @@ jobs:
{% endif %}
PYTHON: "{{ python_version }}"
PYTHON_ABI_TAG: "{{ python_abi_tag }}"
{% if python_version == "3.14" %}
PYTHON_IMAGE_TAG: "3.14"
PYTHON_PATCH_VERSION: "3.14.0"
{% elif python_version == "3.13" %}
PYTHON_IMAGE_TAG: "{{ python_version }}"
PYTHON_PATCH_VERSION: "3.13.7"
{% else %}
PYTHON_IMAGE_TAG: "{{ python_version }}"
{% endif %}

steps:
{{ macros.github_checkout_arrow()|indent }}
{{ macros.github_install_archery()|indent }}
{{ macros.github_login_dockerhub()|indent }}

# Detect free-threaded Python builds
- name: Prepare
run: |
if [ "${PYTHON_ABI_TAG}" = "cp313t" ]; then
if [[ "${PYTHON_ABI_TAG}" == *t ]]; then
test_image_prefix=python-free-threaded
else
test_image_prefix=python
Expand Down
6 changes: 3 additions & 3 deletions dev/tasks/python-wheels/github.osx.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
PYARROW_VERSION: "{{ arrow.no_rc_version }}"
PYTHON_VERSION: "{{ python_version }}"
PYTHON_ABI_TAG: "{{ python_abi_tag }}"
{% if python_abi_tag == "cp313t" %}
{% if python_abi_tag.endswith('t') %}
PYTHON: "/Library/Frameworks/PythonT.framework/Versions/{{ python_version }}/bin/python{{ python_version }}t"
{% else %}
PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}"
Expand Down Expand Up @@ -94,7 +94,7 @@ jobs:
--x-feature=s3

- name: Install Python {{ python_version }}
run: sudo arrow/ci/scripts/install_python.sh macos {{ "3.13t" if python_abi_tag == "cp313t" else python_version }}
run: sudo arrow/ci/scripts/install_python.sh macos {{ python_version + "t" if python_abi_tag.endswith('t') else python_version }}

- name: Build Wheel
run: |
Expand Down Expand Up @@ -131,7 +131,7 @@ jobs:
- name: Test Wheel
env:
PYTEST_ADDOPTS: "-k 'not test_cancellation'"
PYTHON_GIL: {{ 0 if python_abi_tag == "cp313t" else 1 }}
PYTHON_GIL: {{ 0 if python_abi_tag.endswith('t') else 1 }}
run: |
$PYTHON -m venv test-env
source test-env/bin/activate
Expand Down
Loading
Loading