From c8b54dc818269e3cb14ad813a51aabb823bd5986 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 31 Oct 2021 12:04:07 +1100 Subject: [PATCH 1/4] Replaced manylinux1 with manylinux2014 --- .github/workflows/build.yml | 2 +- Makefile | 2 +- .../manylinux1-wheel-build}/Dockerfile | 0 .../manylinux1-wheel-build}/Makefile | 0 .../manylinux1-wheel-build}/README.md | 0 .../manylinux1-wheel-build}/build.sh | 0 .../manylinux1-wheel-build}/build_depends.sh | 0 .../manylinux1-wheel-build}/update.sh | 0 .../manylinux1-wheel-build}/yum_install | 0 manylinux2014-wheel-build/Dockerfile | 23 ++++++++++ manylinux2014-wheel-build/Makefile | 46 +++++++++++++++++++ manylinux2014-wheel-build/README.md | 35 ++++++++++++++ manylinux2014-wheel-build/build.sh | 32 +++++++++++++ manylinux2014-wheel-build/build_depends.sh | 30 ++++++++++++ manylinux2014-wheel-build/update.sh | 2 + manylinux2014-wheel-build/yum_install | 3 ++ 16 files changed, 173 insertions(+), 2 deletions(-) rename {manylinux1-wheel-build => archive/manylinux1-wheel-build}/Dockerfile (100%) rename {manylinux1-wheel-build => archive/manylinux1-wheel-build}/Makefile (100%) rename {manylinux1-wheel-build => archive/manylinux1-wheel-build}/README.md (100%) rename {manylinux1-wheel-build => archive/manylinux1-wheel-build}/build.sh (100%) rename {manylinux1-wheel-build => archive/manylinux1-wheel-build}/build_depends.sh (100%) rename {manylinux1-wheel-build => archive/manylinux1-wheel-build}/update.sh (100%) rename {manylinux1-wheel-build => archive/manylinux1-wheel-build}/yum_install (100%) create mode 100644 manylinux2014-wheel-build/Dockerfile create mode 100644 manylinux2014-wheel-build/Makefile create mode 100644 manylinux2014-wheel-build/README.md create mode 100755 manylinux2014-wheel-build/build.sh create mode 100755 manylinux2014-wheel-build/build_depends.sh create mode 100755 manylinux2014-wheel-build/update.sh create mode 100755 manylinux2014-wheel-build/yum_install diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 73834bb5..7fe9765a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ jobs: - "centos-stream-8-amd64" - "debian-10-buster-x86" - "fedora-34-amd64" - - "manylinux1-wheel-build" + - "manylinux2014-wheel-build" - "ubuntu-18.04-bionic-amd64" - "ubuntu-20.04-focal-amd64" - "ubuntu-20.04-focal-amd64-valgrind" diff --git a/Makefile b/Makefile index 7d947080..b294dbf0 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ TARGETS = \ centos-stream-8-amd64 \ debian-10-buster-x86 \ fedora-34-amd64 \ - manylinux1-wheel-build \ + manylinux2014-wheel-build \ ubuntu-18.04-bionic-amd64 \ ubuntu-20.04-focal-amd64 \ ubuntu-20.04-focal-amd64-valgrind \ diff --git a/manylinux1-wheel-build/Dockerfile b/archive/manylinux1-wheel-build/Dockerfile similarity index 100% rename from manylinux1-wheel-build/Dockerfile rename to archive/manylinux1-wheel-build/Dockerfile diff --git a/manylinux1-wheel-build/Makefile b/archive/manylinux1-wheel-build/Makefile similarity index 100% rename from manylinux1-wheel-build/Makefile rename to archive/manylinux1-wheel-build/Makefile diff --git a/manylinux1-wheel-build/README.md b/archive/manylinux1-wheel-build/README.md similarity index 100% rename from manylinux1-wheel-build/README.md rename to archive/manylinux1-wheel-build/README.md diff --git a/manylinux1-wheel-build/build.sh b/archive/manylinux1-wheel-build/build.sh similarity index 100% rename from manylinux1-wheel-build/build.sh rename to archive/manylinux1-wheel-build/build.sh diff --git a/manylinux1-wheel-build/build_depends.sh b/archive/manylinux1-wheel-build/build_depends.sh similarity index 100% rename from manylinux1-wheel-build/build_depends.sh rename to archive/manylinux1-wheel-build/build_depends.sh diff --git a/manylinux1-wheel-build/update.sh b/archive/manylinux1-wheel-build/update.sh similarity index 100% rename from manylinux1-wheel-build/update.sh rename to archive/manylinux1-wheel-build/update.sh diff --git a/manylinux1-wheel-build/yum_install b/archive/manylinux1-wheel-build/yum_install similarity index 100% rename from manylinux1-wheel-build/yum_install rename to archive/manylinux1-wheel-build/yum_install diff --git a/manylinux2014-wheel-build/Dockerfile b/manylinux2014-wheel-build/Dockerfile new file mode 100644 index 00000000..85cccf83 --- /dev/null +++ b/manylinux2014-wheel-build/Dockerfile @@ -0,0 +1,23 @@ +FROM quay.io/pypa/manylinux2014_x86_64:latest + +run mkdir /src && mkdir /Pillow +ENV SRC=/src +ENV PATH=/opt/python/cp27-cp27m/bin:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +RUN git clone --depth 1 https://github.com/python-pillow/pillow-wheels /src/pillow-wheels + +COPY build_depends.sh /src +copy yum_install /usr/local/bin + +RUN cd $SRC/pillow-wheels \ + && git submodule update --init multibuild \ + && bash $SRC/build_depends.sh + + +FROM quay.io/pypa/manylinux2014_x86_64:latest +COPY --from=0 /usr/local/lib /usr/local/lib +COPY --from=0 /usr/local/include /usr/local/include +RUN yum install -y zlib-devel +COPY build.sh /build.sh + +CMD ["/bin/sh /build.sh"] diff --git a/manylinux2014-wheel-build/Makefile b/manylinux2014-wheel-build/Makefile new file mode 100644 index 00000000..4b0b8429 --- /dev/null +++ b/manylinux2014-wheel-build/Makefile @@ -0,0 +1,46 @@ +WD = $(shell pwd) +USERNAME := $(shell docker info | grep Username | xargs | cut -d ' ' -f 2) +TARGET := $(notdir $(WD)) +ROOT := $(abspath $(WD)/../Pillow) +IMAGENAME := $(if $(USERNAME), $(USERNAME)/$(TARGET), $(TARGET)) +BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +TEST_TARGET = ubuntu-20.04-focal-amd64 +TEST_IMAGE=pythonpillow/$(TEST_TARGET) +.PHONY: build +build: + docker build -t $(IMAGENAME):$(BRANCH) . + +.PHONY: update +update: + ./update.sh + +.PHONY: wheel +wheel: + -[ ! -d out ] && mkdir out + docker run --rm -v $(ROOT):/Pillow -v `pwd`/out:/output $(IMAGENAME):$(BRANCH) /build.sh $(_PYVER) + +36: + _PYVER=36 $(MAKE) wheel +37: + _PYVER=37 $(MAKE) wheel +38: + _PYVER=38 $(MAKE) wheel +39: + _PYVER=39 $(MAKE) wheel + +.PHONY: test +test: 38 + docker run --rm -v $(ROOT):/Pillow -v `pwd`/out:/output $(TEST_IMAGE):$(BRANCH) sh -c "/vpy3/bin/pip install /output/*cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl && cd /Pillow && /vpy3/bin/python selftest.py && /usr/bin/xvfb-run -a /vpy3/bin/pytest -vx" + +.PHONY: push +push: + docker push $(IMAGENAME):$(BRANCH) + +.PHONY: clean +clean: + -rm -r out + +.PHONY: shell +shell: + docker run --rm -it -v $(ROOT):/Pillow $(IMAGENAME):$(BRANCH) /bin/bash + diff --git a/manylinux2014-wheel-build/README.md b/manylinux2014-wheel-build/README.md new file mode 100644 index 00000000..6540227a --- /dev/null +++ b/manylinux2014-wheel-build/README.md @@ -0,0 +1,35 @@ +# Wheel Builder image + +This image is a little different than the other images in this repo -- +it's intended to be a quick way to build manylinux wheels with the +production version of the libraries for debugging and testing +purposes. This is designed to work as a single docker image that +doesn't require privilege or multiple images to run. + +Like the other images in this repo, it expects that there is a Pillow +source directory mounted at /Pillow. Unlike the others, it puts the +output files in /output. This should be mounted as a volume to the host +to retrieve the finished wheel. + +Setting the environment variable `DEBUG=1` will create a build with +symbols for debugging with valgrind/gdb. + +The Makefile has several new commands: + +* make wheel: Makes a python 3.8 manylinux2014 wheel, and puts it in the +./out directory. +* make 36|37|38|39: These are specific commands to make +the corresponding 3.x version in the ./out directory. + +The test target here is mainly to validate the image build, it is +assumed that the builds will be used for other purposes or tested in +other images. . + + +# Sources + +* build_depends is from our integration with oss-fuzz +* yum_install is syntactic sugar to make the multibuild repo work with + the base manylinux wheel image, rather than with it's custom set of + images + diff --git a/manylinux2014-wheel-build/build.sh b/manylinux2014-wheel-build/build.sh new file mode 100755 index 00000000..732a877a --- /dev/null +++ b/manylinux2014-wheel-build/build.sh @@ -0,0 +1,32 @@ +#!/bin/sh -eu + +# options required to trigger the vendored raqm install +OPTS="--global-option build_ext --global-option --vendor-raqm --global-option --vendor-fribidi" + +CFLAGS=${CFLAGS:-} +export CFLAGS="$CFLAGS --std=c99" + +# Check for debug build +DEBUG=${DEBUG:-0} +if [ $DEBUG ]; then + OPTS="--global-option build --global-option --debug $OPTS" + CFLAGS="$CFLAGS -Og -DDEBUG" +fi + +# not strictly necessary, unless running multiple versions from the shell +rm -f /tmp/*.whl || true + +# python version, as 36,37,38,39. Defaults to 3.8. +# Matches the naming in /opt/python/ +PYVER=${1:-38} +PYBIN=$(echo /opt/python/cp${PYVER}*/bin) + +# We have to clean up the pillow directories, otherwise we might get +# cached builds that are not manylinux wheel compatible +cd /Pillow +PATH=$PYBIN:$PATH make clean + +# Build and repair +$PYBIN/pip --verbose wheel ${OPTS} -w /tmp /Pillow +$PYBIN/pip install "auditwheel<5" +$PYBIN/python3 -m auditwheel repair /tmp/Pillow*whl -w /output diff --git a/manylinux2014-wheel-build/build_depends.sh b/manylinux2014-wheel-build/build_depends.sh new file mode 100755 index 00000000..8479ec75 --- /dev/null +++ b/manylinux2014-wheel-build/build_depends.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +. multibuild/common_utils.sh + +export CONFIGURE_BUILD_SOURCED=1 +BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}" +export CPPFLAGS="-I$BUILD_PREFIX/include $CPPFLAGS" +export LIBRARY_PATH="$BUILD_PREFIX/lib:$LIBRARY_PATH" +export PKG_CONFIG_PATH="$BUILD_PREFIX/lib/pkgconfig/:$PKG_CONFIG_PATH" + +. multibuild/library_builders.sh +. config.sh + +yum install -y gcc gcc-c++ +pre_build diff --git a/manylinux2014-wheel-build/update.sh b/manylinux2014-wheel-build/update.sh new file mode 100755 index 00000000..c81a11ea --- /dev/null +++ b/manylinux2014-wheel-build/update.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker pull quay.io/pypa/manylinux2014_x86_64:latest diff --git a/manylinux2014-wheel-build/yum_install b/manylinux2014-wheel-build/yum_install new file mode 100755 index 00000000..f0dc3f0d --- /dev/null +++ b/manylinux2014-wheel-build/yum_install @@ -0,0 +1,3 @@ +#!/bin/sh + +yum install -y $@ From 6d431e0b1088bcf3ff454a4bf3e20ba993ba0fc7 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 31 Oct 2021 12:05:08 +1100 Subject: [PATCH 2/4] Removed Python 3.6 and added Python 3.10 --- manylinux2014-wheel-build/Makefile | 4 ++-- manylinux2014-wheel-build/README.md | 2 +- manylinux2014-wheel-build/build.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/manylinux2014-wheel-build/Makefile b/manylinux2014-wheel-build/Makefile index 4b0b8429..74dd73a4 100644 --- a/manylinux2014-wheel-build/Makefile +++ b/manylinux2014-wheel-build/Makefile @@ -19,14 +19,14 @@ wheel: -[ ! -d out ] && mkdir out docker run --rm -v $(ROOT):/Pillow -v `pwd`/out:/output $(IMAGENAME):$(BRANCH) /build.sh $(_PYVER) -36: - _PYVER=36 $(MAKE) wheel 37: _PYVER=37 $(MAKE) wheel 38: _PYVER=38 $(MAKE) wheel 39: _PYVER=39 $(MAKE) wheel +310: + _PYVER=310 $(MAKE) wheel .PHONY: test test: 38 diff --git a/manylinux2014-wheel-build/README.md b/manylinux2014-wheel-build/README.md index 6540227a..3e2d7fe5 100644 --- a/manylinux2014-wheel-build/README.md +++ b/manylinux2014-wheel-build/README.md @@ -18,7 +18,7 @@ The Makefile has several new commands: * make wheel: Makes a python 3.8 manylinux2014 wheel, and puts it in the ./out directory. -* make 36|37|38|39: These are specific commands to make +* make 37|38|39|310: These are specific commands to make the corresponding 3.x version in the ./out directory. The test target here is mainly to validate the image build, it is diff --git a/manylinux2014-wheel-build/build.sh b/manylinux2014-wheel-build/build.sh index 732a877a..ccb3539a 100755 --- a/manylinux2014-wheel-build/build.sh +++ b/manylinux2014-wheel-build/build.sh @@ -16,7 +16,7 @@ fi # not strictly necessary, unless running multiple versions from the shell rm -f /tmp/*.whl || true -# python version, as 36,37,38,39. Defaults to 3.8. +# python version, as 37,38,39,310. Defaults to 3.8. # Matches the naming in /opt/python/ PYVER=${1:-38} PYBIN=$(echo /opt/python/cp${PYVER}*/bin) From f4e11bd6c184a255aa3ce3b9573cfe22e17a9689 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 31 Oct 2021 16:42:47 +1100 Subject: [PATCH 3/4] Removed Python 2.7 from path --- manylinux2014-wheel-build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manylinux2014-wheel-build/Dockerfile b/manylinux2014-wheel-build/Dockerfile index 85cccf83..edabffbf 100644 --- a/manylinux2014-wheel-build/Dockerfile +++ b/manylinux2014-wheel-build/Dockerfile @@ -2,7 +2,7 @@ FROM quay.io/pypa/manylinux2014_x86_64:latest run mkdir /src && mkdir /Pillow ENV SRC=/src -ENV PATH=/opt/python/cp27-cp27m/bin:/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV PATH=/opt/rh/devtoolset-2/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN git clone --depth 1 https://github.com/python-pillow/pillow-wheels /src/pillow-wheels From 2e10753b8deb2d10568a5ea0cb4adb8fb4a2ab08 Mon Sep 17 00:00:00 2001 From: Andrew Murray <3112309+radarhere@users.noreply.github.com> Date: Sun, 31 Oct 2021 16:45:51 +1100 Subject: [PATCH 4/4] Updated capitalisation and typos Co-authored-by: Hugo van Kemenade --- manylinux2014-wheel-build/README.md | 12 ++++++------ manylinux2014-wheel-build/build.sh | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manylinux2014-wheel-build/README.md b/manylinux2014-wheel-build/README.md index 3e2d7fe5..c4e98c90 100644 --- a/manylinux2014-wheel-build/README.md +++ b/manylinux2014-wheel-build/README.md @@ -3,7 +3,7 @@ This image is a little different than the other images in this repo -- it's intended to be a quick way to build manylinux wheels with the production version of the libraries for debugging and testing -purposes. This is designed to work as a single docker image that +purposes. This is designed to work as a single Docker image that doesn't require privilege or multiple images to run. Like the other images in this repo, it expects that there is a Pillow @@ -12,24 +12,24 @@ output files in /output. This should be mounted as a volume to the host to retrieve the finished wheel. Setting the environment variable `DEBUG=1` will create a build with -symbols for debugging with valgrind/gdb. +symbols for debugging with Valgrind/GDB. The Makefile has several new commands: -* make wheel: Makes a python 3.8 manylinux2014 wheel, and puts it in the +* make wheel: Makes a Python 3.8 manylinux2014 wheel, and puts it in the ./out directory. * make 37|38|39|310: These are specific commands to make the corresponding 3.x version in the ./out directory. The test target here is mainly to validate the image build, it is assumed that the builds will be used for other purposes or tested in -other images. . +other images. # Sources -* build_depends is from our integration with oss-fuzz +* build_depends is from our integration with OSS-Fuzz * yum_install is syntactic sugar to make the multibuild repo work with - the base manylinux wheel image, rather than with it's custom set of + the base manylinux wheel image, rather than with its custom set of images diff --git a/manylinux2014-wheel-build/build.sh b/manylinux2014-wheel-build/build.sh index ccb3539a..4a0b1f7f 100755 --- a/manylinux2014-wheel-build/build.sh +++ b/manylinux2014-wheel-build/build.sh @@ -1,6 +1,6 @@ #!/bin/sh -eu -# options required to trigger the vendored raqm install +# options required to trigger the vendored Raqm install OPTS="--global-option build_ext --global-option --vendor-raqm --global-option --vendor-fribidi" CFLAGS=${CFLAGS:-} @@ -16,12 +16,12 @@ fi # not strictly necessary, unless running multiple versions from the shell rm -f /tmp/*.whl || true -# python version, as 37,38,39,310. Defaults to 3.8. +# Python version, as 37,38,39,310. Defaults to 38. # Matches the naming in /opt/python/ PYVER=${1:-38} PYBIN=$(echo /opt/python/cp${PYVER}*/bin) -# We have to clean up the pillow directories, otherwise we might get +# We have to clean up the Pillow directories, otherwise we might get # cached builds that are not manylinux wheel compatible cd /Pillow PATH=$PYBIN:$PATH make clean