diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 73834bb5..7fe9765a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ jobs: - "centos-stream-8-amd64" - "debian-10-buster-x86" - "fedora-34-amd64" - - "manylinux1-wheel-build" + - "manylinux2014-wheel-build" - "ubuntu-18.04-bionic-amd64" - "ubuntu-20.04-focal-amd64" - "ubuntu-20.04-focal-amd64-valgrind" diff --git a/Makefile b/Makefile index 7d947080..b294dbf0 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ TARGETS = \ centos-stream-8-amd64 \ debian-10-buster-x86 \ fedora-34-amd64 \ - manylinux1-wheel-build \ + manylinux2014-wheel-build \ ubuntu-18.04-bionic-amd64 \ ubuntu-20.04-focal-amd64 \ ubuntu-20.04-focal-amd64-valgrind \ diff --git a/manylinux1-wheel-build/Dockerfile b/archive/manylinux1-wheel-build/Dockerfile similarity index 100% rename from manylinux1-wheel-build/Dockerfile rename to archive/manylinux1-wheel-build/Dockerfile diff --git a/manylinux1-wheel-build/Makefile b/archive/manylinux1-wheel-build/Makefile similarity index 100% rename from manylinux1-wheel-build/Makefile rename to archive/manylinux1-wheel-build/Makefile diff --git a/manylinux1-wheel-build/README.md b/archive/manylinux1-wheel-build/README.md similarity index 100% rename from manylinux1-wheel-build/README.md rename to archive/manylinux1-wheel-build/README.md diff --git a/manylinux1-wheel-build/build.sh b/archive/manylinux1-wheel-build/build.sh similarity index 100% rename from manylinux1-wheel-build/build.sh rename to archive/manylinux1-wheel-build/build.sh diff --git a/manylinux1-wheel-build/build_depends.sh b/archive/manylinux1-wheel-build/build_depends.sh similarity index 100% rename from manylinux1-wheel-build/build_depends.sh rename to archive/manylinux1-wheel-build/build_depends.sh diff --git a/manylinux1-wheel-build/update.sh b/archive/manylinux1-wheel-build/update.sh similarity index 100% rename from manylinux1-wheel-build/update.sh rename to archive/manylinux1-wheel-build/update.sh diff --git a/manylinux1-wheel-build/yum_install b/archive/manylinux1-wheel-build/yum_install similarity index 100% rename from manylinux1-wheel-build/yum_install rename to archive/manylinux1-wheel-build/yum_install diff --git a/manylinux2014-wheel-build/Dockerfile b/manylinux2014-wheel-build/Dockerfile new file mode 100644 index 00000000..edabffbf --- /dev/null +++ b/manylinux2014-wheel-build/Dockerfile @@ -0,0 +1,23 @@ +FROM quay.io/pypa/manylinux2014_x86_64:latest + +run mkdir /src && mkdir /Pillow +ENV SRC=/src +ENV PATH=/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +RUN git clone --depth 1 https://github.com/python-pillow/pillow-wheels /src/pillow-wheels + +COPY build_depends.sh /src +copy yum_install /usr/local/bin + +RUN cd $SRC/pillow-wheels \ + && git submodule update --init multibuild \ + && bash $SRC/build_depends.sh + + +FROM quay.io/pypa/manylinux2014_x86_64:latest +COPY --from=0 /usr/local/lib /usr/local/lib +COPY --from=0 /usr/local/include /usr/local/include +RUN yum install -y zlib-devel +COPY build.sh /build.sh + +CMD ["/bin/sh /build.sh"] diff --git a/manylinux2014-wheel-build/Makefile b/manylinux2014-wheel-build/Makefile new file mode 100644 index 00000000..74dd73a4 --- /dev/null +++ b/manylinux2014-wheel-build/Makefile @@ -0,0 +1,46 @@ +WD = $(shell pwd) +USERNAME := $(shell docker info | grep Username | xargs | cut -d ' ' -f 2) +TARGET := $(notdir $(WD)) +ROOT := $(abspath $(WD)/../Pillow) +IMAGENAME := $(if $(USERNAME), $(USERNAME)/$(TARGET), $(TARGET)) +BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +TEST_TARGET = ubuntu-20.04-focal-amd64 +TEST_IMAGE=pythonpillow/$(TEST_TARGET) +.PHONY: build +build: + docker build -t $(IMAGENAME):$(BRANCH) . + +.PHONY: update +update: + ./update.sh + +.PHONY: wheel +wheel: + -[ ! -d out ] && mkdir out + docker run --rm -v $(ROOT):/Pillow -v `pwd`/out:/output $(IMAGENAME):$(BRANCH) /build.sh $(_PYVER) + +37: + _PYVER=37 $(MAKE) wheel +38: + _PYVER=38 $(MAKE) wheel +39: + _PYVER=39 $(MAKE) wheel +310: + _PYVER=310 $(MAKE) wheel + +.PHONY: test +test: 38 + docker run --rm -v $(ROOT):/Pillow -v `pwd`/out:/output $(TEST_IMAGE):$(BRANCH) sh -c "/vpy3/bin/pip install /output/*cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl && cd /Pillow && /vpy3/bin/python selftest.py && /usr/bin/xvfb-run -a /vpy3/bin/pytest -vx" + +.PHONY: push +push: + docker push $(IMAGENAME):$(BRANCH) + +.PHONY: clean +clean: + -rm -r out + +.PHONY: shell +shell: + docker run --rm -it -v $(ROOT):/Pillow $(IMAGENAME):$(BRANCH) /bin/bash + diff --git a/manylinux2014-wheel-build/README.md b/manylinux2014-wheel-build/README.md new file mode 100644 index 00000000..c4e98c90 --- /dev/null +++ b/manylinux2014-wheel-build/README.md @@ -0,0 +1,35 @@ +# Wheel Builder image + +This image is a little different than the other images in this repo -- +it's intended to be a quick way to build manylinux wheels with the +production version of the libraries for debugging and testing +purposes. This is designed to work as a single Docker image that +doesn't require privilege or multiple images to run. + +Like the other images in this repo, it expects that there is a Pillow +source directory mounted at /Pillow. Unlike the others, it puts the +output files in /output. This should be mounted as a volume to the host +to retrieve the finished wheel. + +Setting the environment variable `DEBUG=1` will create a build with +symbols for debugging with Valgrind/GDB. + +The Makefile has several new commands: + +* make wheel: Makes a Python 3.8 manylinux2014 wheel, and puts it in the +./out directory. +* make 37|38|39|310: These are specific commands to make +the corresponding 3.x version in the ./out directory. + +The test target here is mainly to validate the image build, it is +assumed that the builds will be used for other purposes or tested in +other images. + + +# Sources + +* build_depends is from our integration with OSS-Fuzz +* yum_install is syntactic sugar to make the multibuild repo work with + the base manylinux wheel image, rather than with its custom set of + images + diff --git a/manylinux2014-wheel-build/build.sh b/manylinux2014-wheel-build/build.sh new file mode 100755 index 00000000..4a0b1f7f --- /dev/null +++ b/manylinux2014-wheel-build/build.sh @@ -0,0 +1,32 @@ +#!/bin/sh -eu + +# options required to trigger the vendored Raqm install +OPTS="--global-option build_ext --global-option --vendor-raqm --global-option --vendor-fribidi" + +CFLAGS=${CFLAGS:-} +export CFLAGS="$CFLAGS --std=c99" + +# Check for debug build +DEBUG=${DEBUG:-0} +if [ $DEBUG ]; then + OPTS="--global-option build --global-option --debug $OPTS" + CFLAGS="$CFLAGS -Og -DDEBUG" +fi + +# not strictly necessary, unless running multiple versions from the shell +rm -f /tmp/*.whl || true + +# Python version, as 37,38,39,310. Defaults to 38. +# Matches the naming in /opt/python/ +PYVER=${1:-38} +PYBIN=$(echo /opt/python/cp${PYVER}*/bin) + +# We have to clean up the Pillow directories, otherwise we might get +# cached builds that are not manylinux wheel compatible +cd /Pillow +PATH=$PYBIN:$PATH make clean + +# Build and repair +$PYBIN/pip --verbose wheel ${OPTS} -w /tmp /Pillow +$PYBIN/pip install "auditwheel<5" +$PYBIN/python3 -m auditwheel repair /tmp/Pillow*whl -w /output diff --git a/manylinux2014-wheel-build/build_depends.sh b/manylinux2014-wheel-build/build_depends.sh new file mode 100755 index 00000000..8479ec75 --- /dev/null +++ b/manylinux2014-wheel-build/build_depends.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +. multibuild/common_utils.sh + +export CONFIGURE_BUILD_SOURCED=1 +BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}" +export CPPFLAGS="-I$BUILD_PREFIX/include $CPPFLAGS" +export LIBRARY_PATH="$BUILD_PREFIX/lib:$LIBRARY_PATH" +export PKG_CONFIG_PATH="$BUILD_PREFIX/lib/pkgconfig/:$PKG_CONFIG_PATH" + +. multibuild/library_builders.sh +. config.sh + +yum install -y gcc gcc-c++ +pre_build diff --git a/manylinux2014-wheel-build/update.sh b/manylinux2014-wheel-build/update.sh new file mode 100755 index 00000000..c81a11ea --- /dev/null +++ b/manylinux2014-wheel-build/update.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker pull quay.io/pypa/manylinux2014_x86_64:latest diff --git a/manylinux2014-wheel-build/yum_install b/manylinux2014-wheel-build/yum_install new file mode 100755 index 00000000..f0dc3f0d --- /dev/null +++ b/manylinux2014-wheel-build/yum_install @@ -0,0 +1,3 @@ +#!/bin/sh + +yum install -y $@