ARROW-6920: [Packaging] Build python 3.8 wheels

adds python3.8 wheels

as far as I can tell python3.8 isn't available for Conda yet (conda-forge/python-feedstock#274), so that's will have to be added later

Closes #5685 from sjhewitt/arrow-6920 and squashes the following commits:

feac494 <Sutou Kouhei> Add a TODO comment
908a396 <Sutou Kouhei> Check Docker version for debug
43833a3 <Sutou Kouhei> Add Python 3.8 support to package metadata
f3f9bfa <Krisztián Szűcs> Install tensorflow for 32 bit unicode width py27
cd9ae83 <Sutou Kouhei> Install TensorFlow for Python 2.7 manually
5293f2b <Sutou Kouhei> Pin base image and remove needless virtualenvs
344eecd <Sutou Kouhei> Use apache/arrow-dev again
99cad41 <Sutou Kouhei> Use the uploaded image
f0c8e69 <Sutou Kouhei> Link to libpython only on Windows
260fa79 <Sutou Kouhei> Add a missing empty line
52761b4 <Sutou Kouhei> libpython.so may not exist
5e5977e <Sutou Kouhei> Don't link to other Python libraries on non Windows
e047d54 <Sutou Kouhei> Don't use local function
10dac67 <Terence D. Honles> move plasma subprocesses to top level functions
da300ac <Sutou Kouhei> Don't use TTY
c4526d8 <Sutou Kouhei> Use more recent numpy
7ace0cb <Sutou Kouhei> Ensure using the latest image
655f88c <Sutou Kouhei> Use Azure Pipelines instead of Travis CI
2ef6213 <Sutou Kouhei> Use "call conda.bat activate/deactivate"
1c8427e <Sutou Kouhei> Use conda without call
99251ea <Sutou Kouhei> Use "conda activate/deactivate"
f403b6f <Sutou Kouhei> Use the latest multibuild
c818840 <Sutou Kouhei> Remove m suffix
e9f1408 <Sutou Kouhei> Set up Python 3.8
2ab6732 <Simon Hewitt> ARROW-6920:  Build python 3.8 wheels

Lead-authored-by: Sutou Kouhei <[email protected]>
Co-authored-by: Terence D. Honles <[email protected]>
Co-authored-by: Krisztián Szűcs <[email protected]>
Co-authored-by: Simon Hewitt <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>

Author: 4 people

cpp/src/arrow/python/CMakeLists.txt

@@ -52,10 +52,9 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL
   set_property(SOURCE pyarrow.cc APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-cast-qual ")
 endif()
 
-set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared ${PYTHON_OTHER_LIBS})
-
+set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared)
 if(WIN32)
-  set(ARROW_PYTHON_SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS} ${PYTHON_LIBRARIES})
+  list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_LIBRARIES} ${PYTHON_OTHER_LIBS})
 endif()
 
 set(ARROW_PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS})

dev/tasks/python-wheels/azure.linux.yml

@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+jobs:
+- job: linux
+  pool:
+    vmImage: ubuntu-latest
+  timeoutInMinutes: 360
+  steps:
+    - task: CondaEnvironment@1
+      inputs:
+        packageSpecs: 'click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz'
+        installOptions: '-c conda-forge'
+        updateConda: false
+
+    - script: |
+        set -ex
+        git clone --no-checkout {{ arrow.remote }} arrow
+        git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+        git -C arrow checkout FETCH_HEAD
+        git -C arrow submodule update --init --recursive
+      displayName: Clone arrow
+
+    - script: |
+        set -ex
+        docker -v
+        docker-compose -v
+        cd arrow
+        BUILD_IMAGE=centos-python-{{ wheel_tag }}
+        docker-compose pull $BUILD_IMAGE
+        # TODO(kou): Uncomment this when we resolve "ADD never use cache" problem.
+        # docker-compose build $BUILD_IMAGE
+        docker-compose run \
+          -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \
+          -e PYTHON_VERSION="{{ python_version }}" \
+          -e UNICODE_WIDTH="{{ unicode_width }}" \
+          $BUILD_IMAGE
+      displayName: Build wheel
+
+    # auditwheel does always exit with 0 so it is mostly for debugging
+    # purposes
+    - script: |
+        set -ex
+        cd arrow
+        docker run \
+          -v $(pwd):/arrow quay.io/pypa/{{ wheel_tag }}_x86_64 \
+          /bin/bash -c \
+            "auditwheel show /arrow/python/{{ wheel_tag }}/dist/*.whl"
+      displayName: Audit wheel
+
+    - script: |
+        set -ex
+        cd arrow
+        test_args=""
+        {%- if test_remove_system_libs %}
+        test_args="${test_args} --remove-system-libs"
+        {%- endif %}
+        {%- for image in test_docker_images %}
+        docker run \
+          --shm-size 2G \
+          -v $(pwd):/arrow \
+          -e WHEEL_TAG="{{ wheel_tag }}" \
+          {{ image }} \
+          /arrow/dev/tasks/python-wheels/manylinux-test.sh ${test_args}
+        {%- endfor %}
+      displayName: Test wheel
+
+    - script: |
+        set -ex
+        python arrow/dev/tasks/crossbow.py \
+          --queue-path . \
+          --queue-remote {{ queue.remote_url }} \
+          upload-artifacts \
+          --pattern "arrow/python/{{ wheel_tag }}/dist/*" \
+          --sha {{ task.branch }} \
+          --tag {{ task.tag }}
+      env:
+        CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
+      displayName: Upload packages as a GitHub release

dev/tasks/python-wheels/travis.linux.yml

@@ -31,8 +31,8 @@ env:
     - MACOSX_DEPLOYMENT_TARGET="10.9"
 
 before_install:
-  - git clone https://github.com/matthew-brett/multibuild # TODO pin it
-  - git -C multibuild checkout 4e7a9396e9a50731bb83fc0d16bb98fb0c4032d7
+  - git clone https://github.com/matthew-brett/multibuild
+  - git -C multibuild checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa
 
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}

dev/tasks/python-wheels/travis.osx.yml

@@ -20,14 +20,14 @@
 @rem create conda environment for compiling
 conda update --yes --quiet conda
 
-call conda create -n wheel-build -q -y -c conda-forge ^
+conda create -n wheel-build -q -y -c conda-forge ^
     --file=%ARROW_SRC%\ci\conda_env_cpp.yml ^
     --file=%ARROW_SRC%\ci\conda_env_gandiva.yml ^
     python=%PYTHON_VERSION% ^
     numpy=%NUMPY_VERSION% ^
     || exit /B
 
-call activate wheel-build
+call conda.bat activate wheel-build
 
 @rem Cannot use conda_env_python.yml here because conda-forge has
 @rem ceased providing up-to-date packages for Python 3.5
@@ -84,17 +84,17 @@ pushd %ARROW_SRC%\python
 python setup.py bdist_wheel || exit /B
 popd
 
-call deactivate
+call conda.bat deactivate
 
 set ARROW_TEST_DATA=%ARROW_SRC%\testing\data
 
 @rem test the wheel
 @rem TODO For maximum reliability, we should test in a plain virtualenv instead.
-call conda create -n wheel-test -c conda-forge -q -y ^
+conda create -n wheel-test -c conda-forge -q -y ^
     --file %ARROW_SRC%\ci\conda_env_python.yml ^
     python=%PYTHON_VERSION% ^
     numpy=%NUMPY_VERSION% || exit /B
-call activate wheel-test
+call conda.bat activate wheel-test
 
 @rem install the built wheel
 pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B