ARROW-6920: [Packaging] Build python 3.8 wheels

cpp/src/arrow/python/CMakeLists.txt

@@ -52,10 +52,9 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL
   set_property(SOURCE pyarrow.cc APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-cast-qual ")
 endif()
 
-set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared ${PYTHON_OTHER_LIBS})
-
+set(ARROW_PYTHON_SHARED_LINK_LIBS arrow_shared)
 if(WIN32)
-  set(ARROW_PYTHON_SHARED_LINK_LIBS ${ARROW_PYTHON_SHARED_LINK_LIBS} ${PYTHON_LIBRARIES})
+  list(APPEND ARROW_PYTHON_SHARED_LINK_LIBS ${PYTHON_LIBRARIES} ${PYTHON_OTHER_LIBS})
 endif()
 
 set(ARROW_PYTHON_INCLUDES ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS})

dev/tasks/python-wheels/azure.linux.yml

@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+jobs:
+- job: linux
+  pool:
+    vmImage: ubuntu-latest
+  timeoutInMinutes: 360
+  steps:
+    - task: CondaEnvironment@1
+      inputs:
+        packageSpecs: 'click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz'
+        installOptions: '-c conda-forge'
+        updateConda: false
+
+    - script: |
+        set -ex
+        git clone --no-checkout {{ arrow.remote }} arrow
+        git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+        git -C arrow checkout FETCH_HEAD
+        git -C arrow submodule update --init --recursive
+      displayName: Clone arrow
+
+    - script: |
+        set -ex
+        docker -v
+        docker-compose -v
+        cd arrow
+        BUILD_IMAGE=centos-python-{{ wheel_tag }}
+        docker-compose pull $BUILD_IMAGE
+        # docker-compose build $BUILD_IMAGE
+        docker-compose run \
+          -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \
+          -e PYTHON_VERSION="{{ python_version }}" \
+          -e UNICODE_WIDTH="{{ unicode_width }}" \
+          $BUILD_IMAGE
+      displayName: Build wheel
+
+    # auditwheel does always exit with 0 so it is mostly for debugging
+    # purposes
+    - script: |
+        set -ex
+        cd arrow
+        docker run \
+          -v $(pwd):/arrow quay.io/pypa/{{ wheel_tag }}_x86_64 \
+          /bin/bash -c \
+            "auditwheel show /arrow/python/{{ wheel_tag }}/dist/*.whl"
+      displayName: Audit wheel
+
+    - script: |
+        set -ex
+        cd arrow
+        test_args=""
+        {%- if test_remove_system_libs %}
+        test_args="${test_args} --remove-system-libs"
+        {%- endif %}
+        {%- for image in test_docker_images %}
+        docker run \
+          --shm-size 2G \
+          -v $(pwd):/arrow \
+          -e WHEEL_TAG="{{ wheel_tag }}" \
+          {{ image }} \
+          /arrow/dev/tasks/python-wheels/manylinux-test.sh ${test_args}
+        {%- endfor %}
+      displayName: Test wheel
+
+    - script: |
+        set -ex
+        python arrow/dev/tasks/crossbow.py \
+          --queue-path . \
+          --queue-remote {{ queue.remote_url }} \
+          upload-artifacts \
+          --pattern "arrow/python/{{ wheel_tag }}/dist/*" \
+          --sha {{ task.branch }} \
+          --tag {{ task.tag }}
+      env:
+        CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
+      displayName: Upload packages as a GitHub release

dev/tasks/python-wheels/travis.osx.yml

@@ -31,8 +31,8 @@ env:
     - MACOSX_DEPLOYMENT_TARGET="10.9"
 
 before_install:
-  - git clone https://github.com/matthew-brett/multibuild # TODO pin it
-  - git -C multibuild checkout 4e7a9396e9a50731bb83fc0d16bb98fb0c4032d7
+  - git clone https://github.com/matthew-brett/multibuild
+  - git -C multibuild checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa
 
   - git clone --no-checkout {{ arrow.remote }} arrow
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}

dev/tasks/python-wheels/win-build.bat

@@ -20,14 +20,14 @@
 @rem create conda environment for compiling
 conda update --yes --quiet conda
 
-call conda create -n wheel-build -q -y -c conda-forge ^
+conda create -n wheel-build -q -y -c conda-forge ^
     --file=%ARROW_SRC%\ci\conda_env_cpp.yml ^
     --file=%ARROW_SRC%\ci\conda_env_gandiva.yml ^
     python=%PYTHON_VERSION% ^
     numpy=%NUMPY_VERSION% ^
     || exit /B
 
-call activate wheel-build
+call conda.bat activate wheel-build
 
 @rem Cannot use conda_env_python.yml here because conda-forge has
 @rem ceased providing up-to-date packages for Python 3.5
@@ -84,17 +84,17 @@ pushd %ARROW_SRC%\python
 python setup.py bdist_wheel || exit /B
 popd
 
-call deactivate
+call conda.bat deactivate
 
 set ARROW_TEST_DATA=%ARROW_SRC%\testing\data
 
 @rem test the wheel
 @rem TODO For maximum reliability, we should test in a plain virtualenv instead.
-call conda create -n wheel-test -c conda-forge -q -y ^
+conda create -n wheel-test -c conda-forge -q -y ^
     --file %ARROW_SRC%\ci\conda_env_python.yml ^
     python=%PYTHON_VERSION% ^
     numpy=%NUMPY_VERSION% || exit /B
-call activate wheel-test
+call conda.bat activate wheel-test
 
 @rem install the built wheel
 pip install -vv --no-index --find-links=%ARROW_SRC%\python\dist\ pyarrow || exit /B