manylinux2014

Author: mayeut

.travis.yml

@@ -1,6 +1,6 @@
 language: c
+os: linux
 sudo: required
-dist: trusty
 services:
   - docker
 
@@ -10,48 +10,25 @@ branches:
   except:
     - /^pyup-/
 
-cache:
-  directories:
-    - $HOME/docker
-
 env:
   global:
     # QUAY_USERNAME and QUAY_PASSWORD for docker image upload
     - secure: "lKaTzEL6UNiEfp+BWLOUILG9BMtjwEMpwt6Yag0cQGHix7qJ/ElZ0t3oFw6ZwuDmA5qceAXIdxHLUK9HGVI2MloLk8czGhjvtfJ4XhOxtEJRQ0VkDGPsKN4cfhB4ZjGo6GAPtNqStMyNiY7BZuTrZa7coDLCoUeYcOmTpi6pmd1rrkk725B9QCTuhFHbPhkuL2yu/Jk6WxkHJBKjmuZek+iQa7lRItgMrG0/319PXLvwIGGl00nLFy+Ly5Ciwzux4wuHLTySZQKu0H9FX81A7smM0FW/42kg3ckGa2qLxRw/Pi8Nm/aIk8LD0QXzI5N7HhFfidOTgDS8Mt1HgfxmTk4wUXZ/KvCCshqjimzMc/s9i9wPZX9UqqcfrpZkmwz8dzhm1bndN45ZOCy6xAYT6dzf8T4mLMDjVWSW4+DUoW4sYHRLVujjcMk7ybcwGV43VruPTJnc8XVAhT+VIMQkoPjhQmTOn8h82LRNGYtLa5RReCh9OPKVYB2Quz18FXMWgFt7A6VWudL0c7/8CusLvuo+pLcxt9pnV40rvu1YEohpEj8qR/qTSaDUBZM0J9SVf5zrZR80pZUnXkDF8nm+mcLOTley3YWipU19lCR7dzVyCAiQdVAuNPdnyem3Yk8enGkAJbfLd6eaIDs+p73D0JXh1Nx1px1movVLQH3ohIw="
     - secure: "w1614pomHLltkBhqWM2bOvbymFWIWKqSqqIBDvaNn9tbQScioItJoELBT7g7+cD7nyU7OvpQ1U2fk0xVkCeNvYU0xS1vP4o/VnZRpup7f7Tkiq+2rf4fjwYr3HHnJjwak1l9bsw6FkgzKaVvSdiUJHMVxiIuLd3fVozR7qjBBhTDxSlWGOpSgd+ttpgMZwU5zQjdaVQr1D7E8M0979ZnWMrNRyLiAUeHaPILS815b+ijgqR+i5nmu0/FTCGM9Ik4KIzIfWq8AdfPdbRiq8c+LrrTPfyKcIQJaHmfduYRM4LycGWwzkXFBNtLrJ7uFLG9RDVemOHuHOWIJX8qCUIV4XuESXxH3fUQr6r+yxquTJbzXxNtoaLa6tBOTQWKDrRjT4z9Mf9Im14F2V59EUDoQowHx5bjunOH5wg3ruYNKYYBFRYra5kx0CkKrqFBzyl8fTUEQLyx1HWTVUC1WTXEeD/aFKOSIxW5DxZr5W4LLlW2+Raa52ZzY28Q6AdueFQCRzoJ70/GsJRlSsBdWNOHN4gSp1cZuToLWY15y64QhAMVDpikB+V4hmkbceLiTqeWzTStNL1sa32RHr6i/9zeFZw1pMD1+eOg9x6fgODfh2sqr/zPbu2oONsHnc4D2jwsEax4o+Dv5QHLvK7jdyWUmu47a9QReoexXK60jZXs3CA="
 
-# use YAML aliases and anchors to avoid duplication in stages
-# c.f. https://github.com/travis-ci/travis-ci/issues/8295#issuecomment-454457787
-manylinux-build: &manylinux-build
-      stage: "Build manylinux images"
-      before_install:
-        # Load cached docker images
-        - if [[ -d $HOME/docker ]]; then ls $HOME/docker/*.tar.gz | xargs -I {file} sh -c "zcat {file} | docker load"; fi
-      script:
-        - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh
-      deploy:
-        provider: script
-        script: docker/deploy.sh
-        on:
-          branch: master
-          repo: pypa/manylinux
-
-jobs:
+matrix:
   include:
-    - stage: "Patch glibc"
-      env:
-        - PLATFORM="x86_64"
-      before_install:
-        # Load cached docker images
-        - if [[ -d $HOME/docker ]]; then ls $HOME/docker/*.tar.gz | xargs -I {file} sh -c "zcat {file} | docker load"; fi
-      script:
-        - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh glibc_only
-      before_cache:
-        # Save tagged docker images
-        - mkdir -p $HOME/docker && docker images -a --filter='dangling=false' --format '{{.Repository}}:{{.Tag}} {{.ID}}' | grep 'centos-with-vsyscall:latest' | xargs -n 2 -t sh -c 'test -e $HOME/docker/$1.tar.gz || docker save $0 | gzip -2 > $HOME/docker/$1.tar.gz'
-    - <<: *manylinux-build
-      env:
-        - PLATFORM="x86_64"
-    - <<: *manylinux-build
-      env:
-        - PLATFORM="i686"
+   - arch: amd64
+     env: PLATFORM="x86_64"
+   - arch: arm64
+     env: PLATFORM="aarch64"
+
+script:
+  - PLATFORM=$PLATFORM TRAVIS_COMMIT=$TRAVIS_COMMIT ./build.sh
+
+deploy:
+  provider: script
+  script: docker/deploy.sh
+  on:
+    branch: master
+    repo: pypa/manylinux

build.sh

@@ -4,16 +4,4 @@
 set -ex
 
 
-if [ $PLATFORM == x86_64 ] || [ "$1" == "glibc_only" ]; then
-    # Output something every 10 minutes or Travis kills the job
-    while sleep 9m; do echo -n -e " \b"; done &
-    docker build --rm -t centos-with-vsyscall:latest --cache-from centos-with-vsyscall:latest --target centos-with-vsyscall -f docker/glibc/Dockerfile docker/glibc/
-    # Killing background sleep loop
-    kill %1
-    if [ "$1" == "glibc_only" ]; then
-        exit 0
-    fi
-    docker build --rm -t quay.io/pypa/manylinux2010_centos-6-no-vsyscall --cache-from quay.io/pypa/manylinux2010_centos-6-no-vsyscall:latest --cache-from centos-with-vsyscall:latest -f docker/glibc/Dockerfile docker/glibc/
-fi
-
-docker build --rm -t quay.io/pypa/manylinux2010_$PLATFORM:$TRAVIS_COMMIT -f docker/Dockerfile-$PLATFORM docker/
+docker build --rm -t quay.io/pypa/manylinux2014_$PLATFORM:$TRAVIS_COMMIT -f docker/Dockerfile-$PLATFORM docker/

docker/Dockerfile-aarch64

@@ -0,0 +1,19 @@
+FROM arm64v8/centos:7
+LABEL maintainer="The ManyLinux project"
+
+ENV AUDITWHEEL_ARCH aarch64
+ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH
+ENV LC_ALL C
+ENV LANG C
+ENV LANGUAGE C
+ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-8/root
+ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH
+ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib64:$DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib64/dyninst:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
+ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig
+
+COPY build_scripts /build_scripts
+RUN bash build_scripts/build.sh && rm -r build_scripts
+
+ENV SSL_CERT_FILE=/opt/_internal/certs.pem
+
+CMD ["/bin/bash"]

docker/Dockerfile-i686

@@ -1,9 +1,8 @@
-# See docker/glibc/
-FROM quay.io/pypa/manylinux2010_centos-6-no-vsyscall
+FROM centos:7
 LABEL maintainer="The ManyLinux project"
 
 ENV AUDITWHEEL_ARCH x86_64
-ENV AUDITWHEEL_PLAT manylinux2010_$AUDITWHEEL_ARCH
+ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH
 ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8

docker/Dockerfile-x86_64

@@ -10,14 +10,14 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 
 # Dependencies for compiling Python that we want to remove from
 # the final image after compiling Python
-PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel"
+PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel"
 
-# Libraries that are allowed as part of the manylinux2010 profile
-# Extract from PEP: https://www.python.org/dev/peps/pep-0571/#the-manylinux2010-policy
+# Libraries that are allowed as part of the manylinux2014 profile
+# Extract from PEP: https://www.python.org/dev/peps/pep-0599/#the-manylinux2014-policy
 # On RPM-based systems, they are provided by these packages:
 # Package:    Libraries
 # glib2:      libglib-2.0.so.0, libgthread-2.0.so.0, libgobject-2.0.so.0
-# glibc:      libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libcrypt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6
+# glibc:      libresolv.so.2, libutil.so.1, libnsl.so.1, librt.so.1, libpthread.so.0, libdl.so.2, libm.so.6, libc.so.6
 # libICE:     libICE.so.6
 # libX11:     libX11.so.6
 # libXext:    libXext.so.6
@@ -28,7 +28,7 @@ PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel expat-devel ncurses-devel readline-d
 #
 # PEP is missing the package for libSM.so.6 for RPM based system
 # Install development packages (except for libgcc which is provided by gcc install)
-MANYLINUX2010_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel"
+MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel"
 
 # Get build utilities
 source $MY_DIR/build_utils.sh
@@ -51,42 +51,27 @@ esac
 # Decided not to clean at this point: https://github.com/pypa/manylinux/pull/129
 yum -y update
 
-# EPEL support (for cmake28 & yasm)
-yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm
+# Software collection (for devtoolset-8)
+yum -y install centos-release-scl
 
-case $AUDITWHEEL_ARCH in
-x86_64)
-  # Install devtoolset-8
-  yum -y install centos-release-scl
-  yum -y install \
-    devtoolset-8-binutils \
-    devtoolset-8-gcc \
-    devtoolset-8-gcc-c++ \
-    devtoolset-8-gcc-gfortran \
-  ;;
-i686)
-  # Install devtoolset-7 (binutils, gcc, gcc-c++, gcc-gfortran)
-  devtoolset7s=(
-    "devtoolset-7-runtime-7.1-4.el6.i686.rpm"
-    "devtoolset-7-binutils-2.28-11.el6.i686.rpm"
-    "devtoolset-7-gcc-7.3.1-5.10.el6.i686.rpm"
-    "devtoolset-7-libstdc++-devel-7.3.1-5.10.el6.i686.rpm"
-    "devtoolset-7-gcc-c++-7.3.1-5.10.el6.i686.rpm"
-    "devtoolset-7-libquadmath-devel-7.3.1-5.10.el6.i686.rpm"
-    "devtoolset-7-gcc-gfortran-7.3.1-5.10.el6.i686.rpm"
-  )
-  for rpm in "${devtoolset7s[@]}"; do
-	  yum install -y "https://www.repo.cloudlinux.com/cloudlinux/6.10/sclo/devtoolset-7/i386/$rpm"
-  done
-  ;;
-esac
+if [ "$(uname -m)" == "x86_64" ]; then
+    # EPEL support (for yasm)
+    yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+    YASM=yasm
+elif [ "$(uname -m)" == "aarch64" ]; then
+    # needs erasing to be able to install...
+    yum -y erase centos-release-scl
+fi
 
 # Development tools and libraries
 yum -y install \
     automake \
     bison \
     bzip2 \
-    cmake28 \
+    devtoolset-8-binutils \
+    devtoolset-8-gcc \
+    devtoolset-8-gcc-c++ \
+    devtoolset-8-gcc-gfortran \
     diffutils \
     gettext \
     file \
@@ -96,17 +81,12 @@ yum -y install \
     patch \
     unzip \
     which \
-    yasm \
+    ${YASM} \
     ${PYTHON_COMPILE_DEPS}
 
-# Install a git we link against system OpenSSL/Curl
-yum -y install openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel
+# Install git
 build_git $GIT_ROOT $GIT_HASH
 git version
-yum -y erase openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel perl-devel
-
-# Build an OpenSSL for Pythons. We'll delete this at the end.
-build_openssl $OPENSSL_ROOT $OPENSSL_HASH
 
 # Install newest autoconf
 build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
@@ -155,8 +135,6 @@ ln -s $($PY37_BIN/python -c 'import certifi; print(certifi.where())') \
 # Dockerfiles:
 export SSL_CERT_FILE=/opt/_internal/certs.pem
 
-# Now we can delete our built OpenSSL headers/static libs since we've linked everything we need
-rm -rf /usr/local/ssl
 
 # Install patchelf (latest with unreleased bug fixes)
 curl -fsSL -o patchelf.tar.gz https://github.com/NixOS/patchelf/archive/$PATCHELF_VERSION.tar.gz
@@ -179,7 +157,7 @@ yum -y erase \
     libX11 \
     wireless-tools \
     ${PYTHON_COMPILE_DEPS} > /dev/null 2>&1
-yum -y install ${MANYLINUX2010_DEPS}
+yum -y install ${MANYLINUX_DEPS}
 yum -y clean all > /dev/null 2>&1
 yum list installed
 

docker/build_scripts/build.sh

@@ -1,14 +1,7 @@
 # source me
 
 PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python
-# of the form <maj>.<min>.<rev> or <maj>.<min>.<rev>rc<n>
-CPYTHON_VERSIONS="2.7.16 3.4.10 3.5.7 3.6.9 3.7.5 3.8.0"
-
-# openssl version to build, with expected sha256 hash of .tar.gz
-# archive.
-OPENSSL_ROOT=openssl-1.0.2t
-OPENSSL_HASH=14cb464efe7ac6b54799b34456bd69558a749a4931ecfd9cf9f71d7881cac7bc
-OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source
+CPYTHON_VERSIONS="3.5.7 3.6.9 3.7.5 3.8.0"
 
 PATCHELF_VERSION=0.10
 PATCHELF_HASH=b3cb6bdedcef5607ce34a350cf0b182eb979f8f7bc31eae55a93a70a3f020d13

docker/build_scripts/build_env.sh

@@ -31,27 +31,18 @@ function pyver_dist_dir {
 function do_cpython_build {
     local py_ver=$1
     check_var $py_ver
-    local ucs_setting=$2
-    check_var $ucs_setting
     tar -xzf Python-$py_ver.tgz
     pushd Python-$py_ver
-    if [ "$ucs_setting" = "none" ]; then
-        unicode_flags=""
-        dir_suffix=""
-    else
-        local unicode_flags="--enable-unicode=$ucs_setting"
-        local dir_suffix="-$ucs_setting"
-    fi
-    local prefix="/opt/_internal/cpython-${py_ver}${dir_suffix}"
+    local prefix="/opt/_internal/cpython-${py_ver}"
     mkdir -p ${prefix}/lib
-    ./configure --prefix=${prefix} --disable-shared $unicode_flags > /dev/null
-    make -j2 > /dev/null
-    make install > /dev/null
+    ./configure --prefix=${prefix} --disable-shared > /dev/null
+    make -j$(nproc) > /dev/null
+    make -j$(nproc) install > /dev/null
     popd
     rm -rf Python-$py_ver
     # Some python's install as bin/python3. Make them available as
     # bin/python.
-    if [ -e ${prefix}/bin/python3 ]; then
+    if [ -e ${prefix}/bin/python3 ] && [ ! -e ${prefix}/bin/python ]; then
         ln -s python3 ${prefix}/bin/python
     fi
     ${prefix}/bin/python get-pip.py
@@ -74,12 +65,7 @@ function build_cpython {
     curl -fsSLO $PYTHON_DOWNLOAD_URL/$py_dist_dir/Python-$py_ver.tgz
     curl -fsSLO $PYTHON_DOWNLOAD_URL/$py_dist_dir/Python-$py_ver.tgz.asc
     gpg --verify Python-$py_ver.tgz.asc
-    if [ $(lex_pyver $py_ver) -lt $(lex_pyver 3.3) ]; then
-        do_cpython_build $py_ver ucs2
-        do_cpython_build $py_ver ucs4
-    else
-        do_cpython_build $py_ver none
-    fi
+    do_cpython_build $py_ver
     rm -f Python-$py_ver.tgz
     rm -f Python-$py_ver.tgz.asc
 }
@@ -102,13 +88,6 @@ function build_cpythons {
 }
 
 
-function do_openssl_build {
-    ./config no-ssl2 no-shared -fPIC --prefix=/usr/local/ssl > /dev/null
-    make > /dev/null
-    make install_sw > /dev/null
-}
-
-
 function fetch_source {
     # This is called both inside and outside the build context (e.g. in Travis) to prefetch
     # source tarballs, where curl exists (and works)
@@ -136,19 +115,6 @@ function check_sha256sum {
 }
 
 
-function build_openssl {
-    local openssl_fname=$1
-    check_var ${openssl_fname}
-    local openssl_sha256=$2
-    check_var ${openssl_sha256}
-    fetch_source ${openssl_fname}.tar.gz ${OPENSSL_DOWNLOAD_URL}
-    check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256}
-    tar -xzf ${openssl_fname}.tar.gz
-    (cd ${openssl_fname} && do_openssl_build)
-    rm -rf ${openssl_fname} ${openssl_fname}.tar.gz
-}
-
-
 function build_git {
     local git_fname=$1
     check_var ${git_fname}
@@ -158,15 +124,15 @@ function build_git {
     fetch_source v${git_fname}.tar.gz ${GIT_DOWNLOAD_URL}
     check_sha256sum v${git_fname}.tar.gz ${git_sha256}
     tar -xzf v${git_fname}.tar.gz
-    (cd git-${git_fname} && make install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 > /dev/null)
+    (cd git-${git_fname} && make -j$(nproc) install prefix=/usr/local NO_GETTEXT=1 NO_TCLTK=1 > /dev/null)
     rm -rf git-${git_fname} v${git_fname}.tar.gz
 }
 
 
 function do_standard_install {
     ./configure "$@" > /dev/null
-    make > /dev/null
-    make install > /dev/null
+    make -j$(nproc) > /dev/null
+    make -j$(nproc) install > /dev/null
 }