diff --git a/Makefile b/Makefile index b80cd1a1087..c2df1b5f038 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,12 @@ endif # For the specified GOOS + GOARCH, build all the binaries by default with CGO disabled CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go install -trimpath $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/... # unset GOOS and embed local resources in the vttablet executable - (cd go/cmd/vttablet && unset GOOS && unset GOARCH && go run github.com/GeertJohan/go.rice/rice --verbose append --exec=$${HOME}/go/bin/${GOOS}_${GOARCH}/vttablet) + if [ -d /go/bin ]; then + # Probably in the bootstrap container + (cd go/cmd/vttablet && go run github.com/GeertJohan/go.rice/rice --verbose append --exec=/go/bin/${GOOS}_${GOARCH}/vttablet) + else + (cd go/cmd/vttablet && unset GOOS && unset GOARCH && go run github.com/GeertJohan/go.rice/rice --verbose append --exec=$${HOME}/go/bin/${GOOS}_${GOARCH}/vttablet) + fi # Cross-compiling w/ cgo isn't trivial and we don't need vtorc, so we can skip building it debug: @@ -105,6 +110,13 @@ install: build mkdir -p "$${PREFIX}/bin" cp "$${VTROOT}/bin/"{mysqlctl,mysqlctld,vtorc,vtadmin,vtctld,vtctlclient,vtctldclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/" +# Will only work inside the docker bootstrap for now +cross-install: cross-build + # binaries + mkdir -p "$${PREFIX}/bin" + # Still no vtorc for cross-compile + cp "/go/bin/${GOOS}_${GOARCH}/"{mysqlctl,mysqlctld,vtadmin,vtctld,vtctlclient,vtctldclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/" + # Install local install the binaries needed to run vitess locally # Usage: make install-local PREFIX=/path/to/install/root install-local: build @@ -269,7 +281,14 @@ define build_docker_image ${info Building ${2}} # Fix permissions before copying files, to avoid AUFS bug other must have read/access permissions chmod -R o=rx *; - docker build -f ${1} -t ${2} --build-arg bootstrap_version=${BOOTSTRAP_VERSION} .; + + if grep -q arm64 <<< ${2}; then \ + echo "Building docker using arm64 buildx"; \ + docker buildx build --platform linux/arm64 -f ${1} -t ${2} --build-arg bootstrap_version=${BOOTSTRAP_VERSION} .; \ + else \ + echo "Building docker using straight docker build"; \ + docker build -f ${1} -t ${2} --build-arg bootstrap_version=${BOOTSTRAP_VERSION} .; \ + fi endef docker_base: @@ -285,7 +304,7 @@ docker_base_all: docker_base $(DOCKER_BASE_TARGETS) docker_lite: ${call build_docker_image,docker/lite/Dockerfile,vitess/lite} -DOCKER_LITE_SUFFIX = mysql56 mysql57 ubi7.mysql57 mysql80 ubi7.mysql80 mariadb mariadb103 percona percona57 ubi7.percona57 percona80 ubi7.percona80 alpine testing +DOCKER_LITE_SUFFIX = mysql56 mysql57 ubi7.mysql57 mysql80 ubi7.mysql80 mariadb mariadb103 percona percona57 ubi7.percona57 percona80 ubi7.percona80 alpine testing ubi8.mysql80 ubi8.arm64.mysql80 DOCKER_LITE_TARGETS = $(addprefix docker_lite_,$(DOCKER_LITE_SUFFIX)) $(DOCKER_LITE_TARGETS): docker_lite_%: ${call build_docker_image,docker/lite/Dockerfile.$*,vitess/lite:$*} diff --git a/docker/lite/Dockerfile.ubi8.arm64.mysql80 b/docker/lite/Dockerfile.ubi8.arm64.mysql80 new file mode 100644 index 00000000000..9c2fcd297a3 --- /dev/null +++ b/docker/lite/Dockerfile.ubi8.arm64.mysql80 @@ -0,0 +1,108 @@ +# Copyright 2022 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: We have to build the Vitess binaries from scratch instead of sharing +# a base image because Docker Hub dropped the feature we relied upon to +# ensure images contain the right binaries. + +# Use a temporary layer for the build stage. +ARG bootstrap_version=5 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" + +FROM "${image}" AS builder + +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER + +# Re-copy sources from working tree. +COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess + +# Build and install Vitess in a temporary output directory. +USER vitess +RUN make cross-install PREFIX=/vt/install GOOS=linux GOARCH=arm64 + +# Start over and build the final image. +FROM registry.access.redhat.com/ubi8/ubi:latest + +# Install keys and dependencies +RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ + && yum install -y --setopt=alwaysprompt=no gnupg \ + && ( gpg --keyserver keyserver.ubuntu.com --recv-keys 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A 4D1BB29D63D98E422B2113B19334A25F8507EFA5 99DB70FAE1D7CE227FB6488205B555B38483C65D 3A79BD29 A4A9406876FCBD3C456770C88C718D3B5072E1F5 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1 ) \ + # No xtrabackup packages for aarch64 yet, but still keeping this here + && gpg --export --armor 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A > ${GNUPGHOME}/RPM-GPG-KEY-Percona.1 \ + && gpg --export --armor 4D1BB29D63D98E422B2113B19334A25F8507EFA5 > ${GNUPGHOME}/RPM-GPG-KEY-Percona.2 \ + && gpg --export --armor 99DB70FAE1D7CE227FB6488205B555B38483C65D > ${GNUPGHOME}/RPM-GPG-KEY-CentOS-8 \ + && gpg --export --armor 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1 > ${GNUPGHOME}/RPM-GPG-KEY-EPEL-8 \ + && gpg --export --armor 3A79BD29 > ${GNUPGHOME}/RPM-GPG-KEY-MySQL.1 \ + && gpg --export --armor A4A9406876FCBD3C456770C88C718D3B5072E1F5 > ${GNUPGHOME}/RPM-GPG-KEY-MySQL.2 \ + && rpmkeys --import ${GNUPGHOME}/RPM-GPG-KEY-Percona.1 ${GNUPGHOME}/RPM-GPG-KEY-Percona.2 ${GNUPGHOME}/RPM-GPG-KEY-CentOS-8 ${GNUPGHOME}/RPM-GPG-KEY-MySQL.1 ${GNUPGHOME}/RPM-GPG-KEY-MySQL.2 /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/mysqlrepo.rpm https://dev.mysql.com/get/mysql80-community-release-el8-1.noarch.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/perconarepo.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm \ + && rpmkeys --checksig /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm \ + && rpm -Uvh /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm \ + && rm -f /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm +RUN echo H4sICIDAHmICA2ZvbwDVkDFLxEAQhfv9FVfY7o4RhCBsoXJcIXKHwUIOi7m5MVk2yS6zG0//vYlRULTU4rrHvOHN+2ZL5Q4TP6oeO7bX3Od1pcuFXlyNUzVZg7S2yTmmCwDsgzjuDSUyB5SDI2+QzOChcyJBEnwkPOPQZijNuTkrigKmsHUFJ1MeCjUQEqg61tQweVtM0vOrfXItj1eAM0H0DiR2erTgbnOrV5uVvlk+6M+Kinvctby3p0ptqRziHjOnnxz3s/FnKJcxVlkYu/+k4Zcs+AvM8n3+jWW8MBc2NO6FZILUMEsoYQ76UvWI/vAGB/SOZZsCAAA= | base64 -d | gzip -dc > /etc/yum.repos.d/CentOS-Base.repo \ + && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs --enablerepo c8base --enablerepo c8updates --enablerepo c8extras libev numactl-libs sysstat strace \ + && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs https://download-ib01.fedoraproject.org/pub/epel/8/Everything/aarch64/Packages/g/gperftools-libs-2.7-9.el8.aarch64.rpm https://download-ib01.fedoraproject.org/pub/epel/8/Everything/aarch64/Packages/j/jemalloc-5.2.1-2.el8.aarch64.rpm https://download-ib01.fedoraproject.org/pub/epel/8/Everything/aarch64/Packages/l/libunwind-1.3.1-3.el8.aarch64.rpm +RUN yum update -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs \ + && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs bzip2 ca-certificates gnupg libaio libcurl \ + procps-ng rsync wget openssl hostname curl tzdata make \ +# Can't use alwaysprompt=no here, since we need to pick up deps +# No way to separate key imports and accept deps separately in yum/dnf + && yum install -y --setopt=tsflags=nodocs --enablerepo mysql80-community \ + mysql-community-client mysql-community-server \ +# Have to use hacks to ignore conflicts on /etc/my.cnf install + && mkdir -p /tmp/1 \ +# Enable xtrabackup and percona toolkit repo using the new percona-release tool, commented out for now (no aarch64 packages) + #&& /usr/bin/percona-release enable-only pxb-80 \ + #&& /usr/bin/percona-release enable pt \ + #&& yum install -y --setopt=alwaysprompt=no --downloadonly --downloaddir=/tmp/1 --enablerepo mysql80-community percona-xtrabackup-80 percona-toolkit \ + #&& rpm -Uvh --replacefiles /tmp/1/*rpm \ + && rm -rf /tmp/1 \ + && yum clean all \ + && yum clean all --enablerepo mysql80-community \ + && rm -rf /etc/my.cnf /var/lib/mysql /tmp/gpg /sbin/mysqld-debug + +# Set up Vitess user and directory tree. +RUN groupadd -g 1001 -r vitess && useradd -r -u 1001 -g vitess vitess +RUN mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt + +# Set up Vitess environment (just enough to run pre-built Go binaries) +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot +ENV PATH $VTROOT/bin:$PATH +ENV MYSQL_FLAVOR MySQL80 + +# Copy artifacts from builder layer. +COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator + +RUN mkdir -p /licenses +COPY LICENSE /licenses + +# Create mount point for actual data (e.g. MySQL data dir) +VOLUME /vt/vtdataroot +USER vitess + +LABEL name="Vitess Lite image - MySQL Community Server 8.0" \ + io.k8s.display-name="Vitess Lite image - MySQL Community Server 8.0" \ + maintainer="cncf-vitess-maintainers@lists.cncf.io" \ + vendor="CNCF" \ + version="13.0.0" \ + release="1" \ + summary="Vitess base container image, containing Vitess components along with MySQL Community Server 8.0" \ + description="Vitess base container image, containing Vitess components along with MySQL Community Server 8.0" \ + io.k8s.description="Vitess base container image, containing Vitess components along with MySQL Community Server 8.0" \ + distribution-scope="public" \ + url="https://vitess.io" diff --git a/docker/lite/Dockerfile.ubi8.mysql80 b/docker/lite/Dockerfile.ubi8.mysql80 new file mode 100644 index 00000000000..5aaa8fc2930 --- /dev/null +++ b/docker/lite/Dockerfile.ubi8.mysql80 @@ -0,0 +1,106 @@ +# Copyright 2022 The Vitess Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: We have to build the Vitess binaries from scratch instead of sharing +# a base image because Docker Hub dropped the feature we relied upon to +# ensure images contain the right binaries. + +# Use a temporary layer for the build stage. +ARG bootstrap_version=5 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" + +FROM "${image}" AS builder + +# Allows docker builds to set the BUILD_NUMBER +ARG BUILD_NUMBER + +# Re-copy sources from working tree. +COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess + +# Build and install Vitess in a temporary output directory. +USER vitess +RUN make install PREFIX=/vt/install + +# Start over and build the final image. +FROM registry.access.redhat.com/ubi8/ubi:latest + +# Install keys and dependencies +RUN mkdir /tmp/gpg && chmod 700 /tmp/gpg && export GNUPGHOME=/tmp/gpg \ + && yum install -y --setopt=alwaysprompt=no gnupg \ + && ( gpg --keyserver keyserver.ubuntu.com --recv-keys 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A 4D1BB29D63D98E422B2113B19334A25F8507EFA5 99DB70FAE1D7CE227FB6488205B555B38483C65D 3A79BD29 A4A9406876FCBD3C456770C88C718D3B5072E1F5 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1 ) \ + && gpg --export --armor 430BDF5C56E7C94E848EE60C1C4CBDCDCD2EFD2A > ${GNUPGHOME}/RPM-GPG-KEY-Percona.1 \ + && gpg --export --armor 4D1BB29D63D98E422B2113B19334A25F8507EFA5 > ${GNUPGHOME}/RPM-GPG-KEY-Percona.2 \ + && gpg --export --armor 99DB70FAE1D7CE227FB6488205B555B38483C65D > ${GNUPGHOME}/RPM-GPG-KEY-CentOS-8 \ + && gpg --export --armor 94E279EB8D8F25B21810ADF121EA45AB2F86D6A1 > ${GNUPGHOME}/RPM-GPG-KEY-EPEL-8 \ + && gpg --export --armor 3A79BD29 > ${GNUPGHOME}/RPM-GPG-KEY-MySQL.1 \ + && gpg --export --armor A4A9406876FCBD3C456770C88C718D3B5072E1F5 > ${GNUPGHOME}/RPM-GPG-KEY-MySQL.2 \ + && rpmkeys --import ${GNUPGHOME}/RPM-GPG-KEY-Percona.1 ${GNUPGHOME}/RPM-GPG-KEY-Percona.2 ${GNUPGHOME}/RPM-GPG-KEY-CentOS-8 ${GNUPGHOME}/RPM-GPG-KEY-MySQL.1 ${GNUPGHOME}/RPM-GPG-KEY-MySQL.2 /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release ${GNUPGHOME}/RPM-GPG-KEY-EPEL-8 \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/mysqlrepo.rpm https://dev.mysql.com/get/mysql80-community-release-el8-1.noarch.rpm \ + && curl -L --retry-delay 10 --retry 3 -o /tmp/perconarepo.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm \ + && rpmkeys --checksig /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm \ + && rpm -Uvh /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm \ + && rm -f /tmp/mysqlrepo.rpm /tmp/perconarepo.rpm +RUN echo H4sICIDAHmICA2ZvbwDVkDFLxEAQhfv9FVfY7o4RhCBsoXJcIXKHwUIOi7m5MVk2yS6zG0//vYlRULTU4rrHvOHN+2ZL5Q4TP6oeO7bX3Od1pcuFXlyNUzVZg7S2yTmmCwDsgzjuDSUyB5SDI2+QzOChcyJBEnwkPOPQZijNuTkrigKmsHUFJ1MeCjUQEqg61tQweVtM0vOrfXItj1eAM0H0DiR2erTgbnOrV5uVvlk+6M+Kinvctby3p0ptqRziHjOnnxz3s/FnKJcxVlkYu/+k4Zcs+AvM8n3+jWW8MBc2NO6FZILUMEsoYQ76UvWI/vAGB/SOZZsCAAA= | base64 -d | gzip -dc > /etc/yum.repos.d/CentOS-Base.repo \ + && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs --enablerepo c8base --enablerepo c8updates --enablerepo c8extras libev numactl-libs sysstat strace \ + && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/g/gperftools-libs-2.7-9.el8.x86_64.rpm https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/j/jemalloc-5.2.1-2.el8.x86_64.rpm https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/l/libunwind-1.3.1-3.el8.x86_64.rpm +RUN yum update -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs \ + && yum install -y --setopt=alwaysprompt=no --setopt=tsflags=nodocs bzip2 ca-certificates gnupg libaio libcurl \ + procps-ng rsync wget openssl hostname curl tzdata make \ +# Can't use alwaysprompt=no here, since we need to pick up deps +# No way to separate key imports and accept deps separately in yum/dnf + && yum install -y --setopt=tsflags=nodocs --enablerepo mysql80-community \ + mysql-community-client mysql-community-server \ +# Have to use hacks to ignore conflicts on /etc/my.cnf install + && mkdir -p /tmp/1 \ + && /usr/bin/percona-release enable-only pxb-80 \ + && /usr/bin/percona-release enable pt \ + && yum install -y --setopt=alwaysprompt=no --downloadonly --downloaddir=/tmp/1 --enablerepo mysql80-community percona-xtrabackup-80 percona-toolkit \ + && rpm -Uvh --replacefiles /tmp/1/*rpm \ + && rm -rf /tmp/1 \ + && yum clean all \ + && yum clean all --enablerepo mysql80-community \ + && rm -rf /etc/my.cnf /var/lib/mysql /tmp/gpg /sbin/mysqld-debug + +# Set up Vitess user and directory tree. +RUN groupadd -g 1001 -r vitess && useradd -r -u 1001 -g vitess vitess +RUN mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt + +# Set up Vitess environment (just enough to run pre-built Go binaries) +ENV VTROOT /vt/src/vitess.io/vitess +ENV VTDATAROOT /vt/vtdataroot +ENV PATH $VTROOT/bin:$PATH +ENV MYSQL_FLAVOR MySQL80 + +# Copy artifacts from builder layer. +COPY --from=builder --chown=vitess:vitess /vt/install /vt +COPY --from=builder --chown=vitess:vitess /vt/src/vitess.io/vitess/web/orchestrator /vt/web/orchestrator + +RUN mkdir -p /licenses +COPY LICENSE /licenses + +# Create mount point for actual data (e.g. MySQL data dir) +VOLUME /vt/vtdataroot +USER vitess + +LABEL name="Vitess Lite image - MySQL Community Server 8.0" \ + io.k8s.display-name="Vitess Lite image - MySQL Community Server 8.0" \ + maintainer="cncf-vitess-maintainers@lists.cncf.io" \ + vendor="CNCF" \ + version="13.0.0" \ + release="1" \ + summary="Vitess base container image, containing Vitess components along with MySQL Community Server 8.0" \ + description="Vitess base container image, containing Vitess components along with MySQL Community Server 8.0" \ + io.k8s.description="Vitess base container image, containing Vitess components along with MySQL Community Server 8.0" \ + distribution-scope="public" \ + url="https://vitess.io"