Pin base image and remove needless virtualenvs

Author: kou

dev/tasks/python-wheels/azure.linux.yml

@@ -38,9 +38,8 @@ jobs:
         set -ex
         cd arrow
         BUILD_IMAGE=centos-python-{{ wheel_tag }}
-        # don't attempt to build it, because the base image by pypa is updated
-        # regularly and would cause undeterministic builds
         docker-compose pull $BUILD_IMAGE
+        docker-compose build $BUILD_IMAGE
         docker-compose run \
           -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} \
           -e PYTHON_VERSION="{{ python_version }}" \

python/manylinux1/Dockerfile-x86_64_base

@@ -14,7 +14,10 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-FROM quay.io/pypa/manylinux1_x86_64:latest
+
+# See https://quay.io/repository/pypa/manylinux1_x86_64?tab=history
+# to update base image.
+FROM quay.io/pypa/manylinux1_x86_64@sha256:e7ed385da2363ea796385bb106a98364f7a8a658a5e8dbf1b7e4c71f2f299251
 
 # Install dependencies
 RUN yum install -y xz ccache flex wget && yum clean all
@@ -86,9 +89,6 @@ ENV GLOG_HOME /usr
 WORKDIR /
 RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa
 
-ADD scripts/build_virtualenvs.sh /
-RUN /build_virtualenvs.sh
-
 ADD scripts/build_llvm.sh /
 RUN /build_llvm.sh
 

python/manylinux1/Dockerfile-x86_64_ubuntu

@@ -36,9 +36,6 @@ RUN /install_cmake.sh
 WORKDIR /
 RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa
 
-ADD scripts/build_virtualenvs.sh /
-RUN /build_virtualenvs.sh
-
 ADD scripts/build_openssl.sh /
 RUN /build_openssl.sh
 

python/manylinux1/build_arrow.sh

@@ -58,14 +58,17 @@ export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost -
 mkdir -p /io/dist
 
 # Must pass PYTHON_VERSION and UNICODE_WIDTH env variables
-# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16
+# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 3.8,16
 
 CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})"
 PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python"
 PIP="${CPYTHON_PATH}/bin/pip"
 # Put our Python first to avoid picking up an antiquated Python from CMake
 PATH="${CPYTHON_PATH}/bin:${PATH}"
 
+echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
+$PIP install -r requirements-wheel.txt
+
 if [ "${PYTHON_VERSION}" != "2.7" ]; then
   export PYARROW_WITH_FLIGHT=1
   export PYARROW_WITH_GANDIVA=1
@@ -123,9 +126,6 @@ popd
 # Check that we don't expose any unwanted symbols
 /io/scripts/check_arrow_visibility.sh
 
-echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
-$PIP install -r requirements-wheel.txt
-
 # Clear output directories and leftovers
 rm -rf dist/
 rm -rf build/

python/manylinux1/scripts/build_virtualenvs.sh

@@ -15,7 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM quay.io/pypa/manylinux2010_x86_64:latest
+# See https://quay.io/repository/pypa/manylinux2010_x86_64?tab=history
+# to update base image.
+FROM quay.io/pypa/manylinux2010_x86_64@sha256:d41631c7360a0028876755aebb7036db639c24e7dcdaf3a9e6dcc0e74a681541
 
 # Install build dependencies
 RUN yum install -y xz bison ccache flex wget
@@ -94,6 +96,3 @@ RUN /build_re2.sh
 
 ADD scripts/build_bz2.sh /
 RUN /build_bz2.sh
-
-ADD scripts/build_virtualenvs.sh /
-RUN /build_virtualenvs.sh

python/manylinux2010/Dockerfile-x86_64_base

@@ -58,13 +58,16 @@ export PYARROW_CMAKE_OPTIONS='-DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_
 mkdir -p /io/dist
 
 # Must pass PYTHON_VERSION and UNICODE_WIDTH env variables
-# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16
+# possible values are: 2.7,16 2.7,32 3.5,16 3.6,16 3.7,16 3.8,16
 
 CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})"
 PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python"
 PIP="${CPYTHON_PATH}/bin/pip"
 PATH="${PATH}:${CPYTHON_PATH}"
 
+echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
+$PIP install -r requirements-wheel.txt
+
 if [ "${PYTHON_VERSION}" != "2.7" ]; then
   export PYARROW_WITH_FLIGHT=1
   export PYARROW_WITH_GANDIVA=1
@@ -123,9 +126,6 @@ popd
 # Check that we don't expose any unwanted symbols
 /io/scripts/check_arrow_visibility.sh
 
-echo "=== (${PYTHON_VERSION}) Install the wheel build dependencies ==="
-$PIP install -r requirements-wheel.txt
-
 # Clear output directories and leftovers
 rm -rf dist/
 rm -rf build/

python/manylinux2010/build_arrow.sh

@@ -1,6 +1,10 @@
+Keras-Preprocessing
 cython
-wheel==0.31.1
+futures; python_version < "3.2"
+numpy>=1.16
+pandas
 setuptools_scm==3.2.0
 six>=1.0.0
-numpy>=1.16
-futures; python_version < "3.2"
+# TODO: TensorFlow doesn't support Python 3.8 yet.
+tensorflow; python_version < "3.8"
+wheel==0.31.1