From ddcb2b902add101eaf066e8df6d4a1f84b0f1ee0 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 3 Sep 2021 12:25:03 +0100 Subject: [PATCH 01/11] dist: Add debian packaging Signed-off-by: Ryan Northey --- .azure-pipelines/pipelines.yml | 68 +++++++++++ ci/do_ci.sh | 51 +++++++- distribution/BUILD | 38 ++++++ distribution/debian/BUILD | 10 ++ distribution/debian/copyright | 8 ++ distribution/debian/packages.bzl | 109 ++++++++++++++++++ distribution/debian/preinst | 65 +++++++++++ distribution/distros.yaml | 24 ++++ .../distrotest.sh | 5 +- distribution/packages.bzl | 65 +++++++++++ distribution/verification.sh | 12 ++ tools/base/requirements.in | 8 +- tools/base/requirements.txt | 39 ++++--- tools/distribution/BUILD | 4 - 14 files changed, 482 insertions(+), 24 deletions(-) create mode 100644 distribution/BUILD create mode 100644 distribution/debian/BUILD create mode 100644 distribution/debian/copyright create mode 100644 distribution/debian/packages.bzl create mode 100755 distribution/debian/preinst create mode 100644 distribution/distros.yaml rename {tools/distribution => distribution}/distrotest.sh (93%) create mode 100644 distribution/packages.bzl create mode 100755 distribution/verification.sh diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index c16691ca580e4..f6d5a660b2b04 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -233,6 +233,74 @@ stages: artifactSuffix: ".arm64" bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" +- stage: distribution + dependsOn: ["linux_arm64", "linux_x64"] + jobs: + - job: package_x64 + dependsOn: [] + timeoutInMinutes: 120 + pool: + vmImage: "ubuntu-18.04" + steps: + - template: bazel.yml + parameters: + ciTarget: bazel.distribution + - job: package_arm64 + dependsOn: [] + timeoutInMinutes: 120 + pool: "arm-large" + steps: + - template: bazel.yml + parameters: + managedAgent: false + ciTarget: bazel.distribution + rbe: false + artifactSuffix: ".arm64" + bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" + - job: verify_x64 + dependsOn: ["package_x64"] + timeoutInMinutes: 120 + pool: + vmImage: "ubuntu-18.04" + steps: + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.distribution" + itemPattern: "bazel.distribution/packages.x64.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh verify_distro' + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Verify packages" + - job: verify_arm64 + dependsOn: ["package_arm64"] + timeoutInMinutes: 120 + pool: "arm-large" + steps: + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.distribution.arm64" + itemPattern: "bazel.distribution.arm64/packages.arm64.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh verify_distro' + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Verify packages" + - stage: check dependsOn: ["linux_x64"] jobs: diff --git a/ci/do_ci.sh b/ci/do_ci.sh index d7a0c90e79348..3f183f9880ccd 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -7,8 +7,9 @@ set -e build_setup_args="" if [[ "$1" == "format_pre" || "$1" == "fix_format" || "$1" == "check_format" || "$1" == "docs" || \ - "$1" == "bazel.clang_tidy" || "$1" == "tooling" || "$1" == "deps" || "$1" == "verify_examples" || \ - "$1" == "verify_build_examples" ]]; then + "$1" == "bazel.clang_tidy" || "$1" == "bazel.distribution" || "$1" == "tooling" \ + || "$1" == "deps" || "$1" == "verify_examples" || "$1" == "verify_build_examples" \ + || "$1" == "verify_distro" ]]; then build_setup_args="-nofetch" fi @@ -214,6 +215,44 @@ if [[ "$CI_TARGET" == "bazel.release" ]]; then echo "bazel contrib release build..." bazel_contrib_binary_build release + exit 0 +elif [[ "$CI_TARGET" == "bazel.distribution" ]]; then + setup_clang_toolchain + + if ! gpg --list-secret-keys "*"; then + # create a throwaway key to sign the packages with + # packages will otherwise be signed by the first available key + KEYDATA=$(cat < + +Files: * +Copyright: Copyright 2016-2018 Envoy Project Authors +License: Apache +/usr/share/common-licenses/Apache-2.0 diff --git a/distribution/debian/packages.bzl b/distribution/debian/packages.bzl new file mode 100644 index 0000000000000..5b96dd6df2739 --- /dev/null +++ b/distribution/debian/packages.bzl @@ -0,0 +1,109 @@ +load("@rules_pkg//:pkg.bzl", "pkg_deb", "pkg_tar") + +GLIBC_MIN_VERSION = "2.27" + +def envoy_pkg_deb( + name, + data, + homepage = "https://www.envoyproxy.io/", + description = "Envoy built for Debian/Ubuntu", + preinst = "//distribution/debian:preinst", + supported_distributions = "buster bullseye bionic focal hirstute impish", + architecture = select({ + "//bazel:x86": "amd64", + "//conditions:default": "arm64", + }), + depends = [ + "libc6 (>= %s)" % GLIBC_MIN_VERSION, + ], + version = None, + maintainer = None, + **kwargs): + """Wrapper for `pkg_deb` with Envoy defaults""" + pkg_deb( + name = "%s.deb" % name, + architecture = architecture, + data = data, + depends = depends, + description = description, + distribution = supported_distributions, + homepage = homepage, + maintainer = maintainer, + package = name, + version = version, + preinst = preinst, + **kwargs + ) + +def envoy_pkg_debs(name, version, release_version, maintainer, bin_files = ":envoy-bin-files", config = ":envoy-config"): + """Package the Envoy .debs with their .changes files. + + Packages are created for the version *and* the release version, eg + + - envoy_1.21.0_amd64.deb + - envoy-1.21_1.21.0_amd64.deb + + This way packages are available for both "envoy" and "envoy-1.21" in package managers. + """ + + # generate deb data for all packages + pkg_tar( + name = "deb-data", + srcs = [ + "//distribution/debian:copyright", + config, + bin_files, + ], + remap_paths = {"/copyright": "/usr/share/doc/envoy/copyright"}, + ) + + # generate package for this patch version + envoy_pkg_deb( + name = "envoy", + data = ":deb-data", + version = version, + maintainer = maintainer, + ) + + # generate package for this minor version + envoy_pkg_deb( + name = "envoy-%s" % release_version, + data = ":deb-data", + version = version, + conflicts = ["envoy"], + provides = ["envoy"], + maintainer = maintainer, + ) + + # TODO(phlax): Remove this hack + # Due to upstream issues with `OutputGroupInfo` files from `pkg_deb` + # we have to follow the real filepath of the .deb file to find the + # .changes file (~artefact) + # For this hack to work, strategy needs to be `sandbox,local` for the + # following mnemonics: + # - Genrule + # - MakeDeb + # + # Upstream issue is here: https://github.com/bazelbuild/rules_pkg/issues/477 + # + + deb_files = ( + "envoy_deb1=$$(realpath $(location :envoy.deb)) " + + "&& envoy_deb2=$$(realpath $(location :envoy-%s.deb)) \\" % release_version + ) + + # bundle all debs and changes files into /debs folder of tarball + native.genrule( + name = name, + srcs = ["envoy.deb", "envoy-%s.deb" % release_version], + outs = [":debs.tar"], + cmd = deb_files + """ + && tar --transform "flags=r;s|^|deb/|" \ + -C $$(dirname $$envoy_deb1) \ + -cf $@ \ + $$(basename $${envoy_deb1}) \ + $$(basename $${envoy_deb1%.deb}.changes) \ + $$(basename $${envoy_deb2}) \ + $$(basename $${envoy_deb2%.deb}.changes) + """, + ) diff --git a/distribution/debian/preinst b/distribution/debian/preinst new file mode 100755 index 0000000000000..6e67a52313bac --- /dev/null +++ b/distribution/debian/preinst @@ -0,0 +1,65 @@ +#! /bin/sh + +# preinst script for envoy + +set -e + +addenvoyuser() { + if ! getent group envoy >/dev/null; then + addgroup --system envoy >/dev/null + fi + + if ! getent passwd envoy >/dev/null; then + adduser \ + --system \ + --disabled-login \ + --ingroup envoy \ + --no-create-home \ + --home /nonexistent \ + --gecos "envoy user" \ + --shell /bin/false \ + envoy >/dev/null + + fi +} + + +case "$1" in + + install) + addenvoyuser + cat <&2 + exit 0 + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/distribution/distros.yaml b/distribution/distros.yaml new file mode 100644 index 0000000000000..c02180d91662a --- /dev/null +++ b/distribution/distros.yaml @@ -0,0 +1,24 @@ + +debian_buster: + image: debian:buster-slim + ext: buster.changes + +debian_bullseye: + image: debian:bullseye-slim + ext: bullseye.changes + +ubuntu_bionic: + image: ubuntu:18.04 + ext: bionic.changes + +ubuntu_focal: + image: ubuntu:20.04 + ext: focal.changes + +ubuntu_hirstute: + image: ubuntu:21.04 + ext: hirstute.changes + +ubuntu_impish: + image: ubuntu:21.10 + ext: impish.changes diff --git a/tools/distribution/distrotest.sh b/distribution/distrotest.sh similarity index 93% rename from tools/distribution/distrotest.sh rename to distribution/distrotest.sh index 0b48de956a213..e29cb6483b903 100755 --- a/tools/distribution/distrotest.sh +++ b/distribution/distrotest.sh @@ -84,7 +84,8 @@ test "$(stat -L -c "%a %G %U" /usr/bin/envoy)" == "$BINARY_PERMISSIONS" && echo run_log config-permissions "Check ownership/permissons of envoy config" test "$(stat -L -c "%a %G %U" /etc/envoy/envoy.yaml)" == "$CONFIG_PERMISSIONS" && echo "Correct permissions: ${CONFIG_PERMISSIONS}" -run_log envoy-version "Envoy version" +run_log envoy-version "Envoy version: ${ENVOY_VERSION}" +envoy --version envoy --version | grep "$ENVOY_VERSION" run_log start-envoy "Start Envoy" @@ -99,7 +100,7 @@ pgrep envoy run_log proxy-responds "Check proxy responds" RESPONSE=$(curl -s http://localhost:10000/) -echo "$RESPONSE" | grep "Welcome to Envoy" +echo "$RESPONSE" | grep "Envoy is an open source edge and service proxy, designed for cloud-native applications" run_log stop-envoy "Stop envoy" sudo -u envoy pkill envoy && echo "Envoy stopped" diff --git a/distribution/packages.bzl b/distribution/packages.bzl new file mode 100644 index 0000000000000..7477574267a22 --- /dev/null +++ b/distribution/packages.bzl @@ -0,0 +1,65 @@ +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_files") +load("//distribution/debian:packages.bzl", "envoy_pkg_debs") + +def _release_version_for(version): + if "-" in version: + version, version_suffix = version.split("-") + + major, minor, patch = version.split(".") + return ".".join((major, minor)) + +def envoy_pkg_distros( + name, + envoy_bin = ":envoy-bin", + version = None, + maintainer = None, + config = "//configs:envoyproxy_io_proxy.yaml"): + # data common to all packages + pkg_files( + name = "envoy-config", + srcs = [config], + renames = { + config: "/etc/envoy/envoy.yaml", + }, + ) + + pkg_files( + name = "envoy-bin-files", + srcs = [envoy_bin], + attributes = pkg_attributes(mode = "0755"), + renames = {envoy_bin: "/usr/bin/envoy"}, + ) + + # build debs + envoy_pkg_debs( + name = "debs", + version = version, + release_version = _release_version_for(version), + maintainer = maintainer, + ) + + # bundle distro packages into a tarball + pkg_tar( + name = "distro_packages", + extension = "tar", + deps = [ + ":debs", + ], + ) + + # sign the packages + native.genrule( + name = name, + cmd = """ + $(location //tools/distribution:sign) \ + --extract \ + --tar $@ \ + $(location :distro_packages) + """, + outs = ["%s.tar.gz" % name], + srcs = [":distro_packages"], + tools = [ + "//tools/distribution:sign", + ], + ) diff --git a/distribution/verification.sh b/distribution/verification.sh new file mode 100755 index 0000000000000..a5fc4e594f0f4 --- /dev/null +++ b/distribution/verification.sh @@ -0,0 +1,12 @@ +#!/bin/bash -e + +echo "$@" + +VERIFY_COMMAND="$1" +TESTFILE="$2" +DISTRO_TAR="$3" +PACKAGE_BUILD="$4" + +shift 4 + +"$VERIFY_COMMAND" "$TESTFILE" "$DISTRO_TAR" "$PACKAGE_BUILD" "${@}" diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 8041aa7fb3d78..749abde76067e 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -6,14 +6,16 @@ coloredlogs coverage envoy.base.checker envoy.base.runner -envoy.base.utils>=0.0.10 +envoy.base.utils>=0.0.12 envoy.code_format.python_check>=0.0.4 envoy.dependency.cve_scan envoy.dependency.pip_check>=0.0.4 +envoy.distribution.distrotest>=0.0.5 envoy.distribution.release -envoy.distribution.verify +envoy.distribution.verify>=0.0.6 envoy.docs.sphinx-runner>=0.0.3 -envoy.gpg.sign +envoy.gpg.identity>=0.0.3 +envoy.gpg.sign>=0.0.5 flake8 frozendict gitpython diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 716b6135722c9..02f56485623c6 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -44,6 +44,7 @@ aiodocker==0.21.0 \ --hash=sha256:6fe00135bb7dc40a407669d3157ecdfd856f3737d939df54f40a479d40cf7bdc # via # envoy.distribution.distrotest + # envoy.distribution.verify # envoy.docker.utils aiofiles==0.7.0 \ --hash=sha256:a1c4fc9b2ff81568c83e21392a82f344ea9d23da906e4f6a52662764545e19d4 \ @@ -278,15 +279,16 @@ envoy.base.runner==0.0.4 \ # envoy.docs.sphinx-runner # envoy.github.abstract # envoy.gpg.sign -envoy.base.utils==0.0.10 \ - --hash=sha256:252f47a17c6346b2496c24c239e7ff7b1312a62f0605877c5315176d3387aa61 \ - --hash=sha256:a3e9fe045c2f117e4b5754c0f412a46ff1e058796e42aabda416f52029c7cd53 +envoy.base.utils==0.0.12 \ + --hash=sha256:0c365ad4eb38752866dcfed0c1a09fc4343eb1efc3507e67eee461f580d734ae \ + --hash=sha256:e23c5c7931bfd1ca93e9538d69534d616980bf0f7bf29c32da3ee1e05ba4793d # via # -r requirements.in # envoy.code-format.python-check # envoy.dependency.cve-scan # envoy.dependency.pip-check # envoy.distribution.distrotest + # envoy.distribution.verify # envoy.docs.sphinx-runner # envoy.github.release # envoy.gpg.sign @@ -300,14 +302,18 @@ envoy.dependency.cve-scan==0.0.1 \ envoy.dependency.pip-check==0.0.4 \ --hash=sha256:3213d77959f65c3c97e9b5d74cb14c02bc02dae64bac2e7c3cb829a2f4e5e40e # via -r requirements.in -envoy.distribution.distrotest==0.0.3 \ - --hash=sha256:c094adbd959eb1336f93afc00aedb7ee4e68e8252e2365be816a6f9ede8a3de7 - # via envoy.distribution.verify +envoy.distribution.distrotest==0.0.5 \ + --hash=sha256:042747ff9691fe33bfccbf7821bae5735b705b9488a9a97f702d6b5b37063245 \ + --hash=sha256:ea9d7cd819c0dbc34bc412c8c6608cb2898b002517ff32c3baa2a1b5275e93fb + # via + # -r requirements.in + # envoy.distribution.verify envoy.distribution.release==0.0.4 \ --hash=sha256:41037e0488f0593ce5173739fe0cd1b45a4775f5a47738b85d9d04024ca241a2 # via -r requirements.in -envoy.distribution.verify==0.0.2 \ - --hash=sha256:ae59134085de50203edf51c243dbf3301cbe5550db29f0ec6f9ea1c3b82fee1c +envoy.distribution.verify==0.0.6 \ + --hash=sha256:7b5022ca18189089f24dd31f0486a470e1a6636a8cc352a0507ca6305421f932 \ + --hash=sha256:dbb7e535a5c18e0845b6d66017e3ca816254f40b2bc025951d92c5059ba9d518 # via -r requirements.in envoy.docker.utils==0.0.2 \ --hash=sha256:a12cb57f0b6e204d646cbf94f927b3a8f5a27ed15f60d0576176584ec16a4b76 @@ -324,11 +330,15 @@ envoy.github.abstract==0.0.16 \ envoy.github.release==0.0.8 \ --hash=sha256:fbc4354030137eb565b8c4d679965e4ef60b01de0c09310441836e592ca0cd19 # via envoy.distribution.release -envoy.gpg.identity==0.0.2 \ - --hash=sha256:7d32ff9133e00b9974b4dabd2512b4872b091b8c5069d0112240dcc1a56bc406 - # via envoy.gpg.sign -envoy.gpg.sign==0.0.3 \ - --hash=sha256:31667931f5d7ff05fd809b89748f277511486311c777652af4cb8889bd641049 +envoy.gpg.identity==0.0.3 \ + --hash=sha256:26fb7fae674d252da67bf4abac7db948824219840ae5864eb9e5b871f665613a \ + --hash=sha256:46b2a36ee7cdd643db7cad945a119a71f84eec3c6237635bb135d510a86169dc + # via + # -r requirements.in + # envoy.gpg.sign +envoy.gpg.sign==0.0.5 \ + --hash=sha256:1d1b844d481f34f678e177bc0a0d1d7429d9e3424eade2166f21157ebe638b12 \ + --hash=sha256:65e56e0e7d6158ffcbfa2d85c18398dce83fe715cd7a64f327291dcab6332914 # via -r requirements.in flake8==4.0.1 \ --hash=sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d \ @@ -744,6 +754,7 @@ typing-extensions==3.10.0.2 \ # via # aiodocker # aiohttp + # gitpython uritemplate==3.0.1 \ --hash=sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f \ --hash=sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae @@ -758,8 +769,10 @@ verboselogs==1.7 \ # via # -r requirements.in # envoy.base.runner + # envoy.distribution.distrotest # envoy.github.abstract # envoy.github.release + # envoy.gpg.sign wrapt==1.12.1 \ --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7 # via deprecated diff --git a/tools/distribution/BUILD b/tools/distribution/BUILD index e00e257bdc4ea..ad6591fc0102d 100644 --- a/tools/distribution/BUILD +++ b/tools/distribution/BUILD @@ -5,10 +5,6 @@ licenses(["notice"]) # Apache 2 envoy_package() -exports_files([ - "distrotest.sh", -]) - alias( name = "release", actual = entry_point( From 0ac11cc7a6cbde317a6246049628dd024bdaa59c Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 3 Dec 2021 13:47:57 +0000 Subject: [PATCH 02/11] update Signed-off-by: Ryan Northey --- distribution/verification.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/distribution/verification.sh b/distribution/verification.sh index a5fc4e594f0f4..5220268f0c662 100755 --- a/distribution/verification.sh +++ b/distribution/verification.sh @@ -4,9 +4,14 @@ echo "$@" VERIFY_COMMAND="$1" TESTFILE="$2" -DISTRO_TAR="$3" -PACKAGE_BUILD="$4" +VERSION="$3" +CONFIG="$4" shift 4 -"$VERIFY_COMMAND" "$TESTFILE" "$DISTRO_TAR" "$PACKAGE_BUILD" "${@}" +echo "VERIFY_COMMAND: ${VERIFY_COMMAND}" +echo "TESTFILE: ${TESTFILE}" +echo "VERSION: ${VERSION}" +echo "CONFIG: ${CONFIG}" + +"$VERIFY_COMMAND" "$TESTFILE" "$VERSION" "$CONFIG" "${@}" From 869db6e1a05656ef2dde1d60430ed81dadbdc6a2 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Tue, 7 Dec 2021 20:40:54 +0000 Subject: [PATCH 03/11] upstream-gen-key Signed-off-by: Ryan Northey --- ci/do_ci.sh | 30 +++++++++++------------------- distribution/packages.bzl | 13 ++++++++++++- tools/base/requirements.in | 4 ++-- tools/base/requirements.txt | 12 ++++++------ 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 3f183f9880ccd..91b11ad472f5c 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -217,30 +217,22 @@ if [[ "$CI_TARGET" == "bazel.release" ]]; then exit 0 elif [[ "$CI_TARGET" == "bazel.distribution" ]]; then + echo "Building distro packages..." + setup_clang_toolchain + # By default the packages will be signed by the first available key. + # If there is no key available, a throwaway key is created + # and the packages signed with it, for the purpose of testing only. if ! gpg --list-secret-keys "*"; then - # create a throwaway key to sign the packages with - # packages will otherwise be signed by the first available key - KEYDATA=$(cat <=0.0.5 envoy.distribution.release envoy.distribution.verify>=0.0.6 envoy.docs.sphinx-runner>=0.0.3 -envoy.gpg.identity>=0.0.3 -envoy.gpg.sign>=0.0.5 +envoy.gpg.identity>=0.0.6 +envoy.gpg.sign>=0.0.7 flake8 frozendict gitpython diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 02f56485623c6..788e7f15e4ab5 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -330,15 +330,15 @@ envoy.github.abstract==0.0.16 \ envoy.github.release==0.0.8 \ --hash=sha256:fbc4354030137eb565b8c4d679965e4ef60b01de0c09310441836e592ca0cd19 # via envoy.distribution.release -envoy.gpg.identity==0.0.3 \ - --hash=sha256:26fb7fae674d252da67bf4abac7db948824219840ae5864eb9e5b871f665613a \ - --hash=sha256:46b2a36ee7cdd643db7cad945a119a71f84eec3c6237635bb135d510a86169dc +envoy.gpg.identity==0.0.6 \ + --hash=sha256:2195e09aaacbaa8d6588378ad96d206708ffaf863aead38445eb3e54e4223c34 \ + --hash=sha256:93032dd82c9c422c2145dbf426fd93c3673c53ba03cca2eab13f5fb8cf24f47c # via # -r requirements.in # envoy.gpg.sign -envoy.gpg.sign==0.0.5 \ - --hash=sha256:1d1b844d481f34f678e177bc0a0d1d7429d9e3424eade2166f21157ebe638b12 \ - --hash=sha256:65e56e0e7d6158ffcbfa2d85c18398dce83fe715cd7a64f327291dcab6332914 +envoy.gpg.sign==0.0.7 \ + --hash=sha256:898907dc76627802ab248576d2fb7d12ed4cce75148b5b070382bb51d25e6504 \ + --hash=sha256:d61165d631d92c9726129ae72ddbb409d2514de3540a829d9ed9a090c165459e # via -r requirements.in flake8==4.0.1 \ --hash=sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d \ From 1c51ab5af411eada3b042804d404ae5ab684e72e Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 16 Dec 2021 10:17:23 +0000 Subject: [PATCH 04/11] rules_pkg Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 8 ++--- ci/do_ci.sh | 2 -- distribution/debian/packages.bzl | 55 +++++++++++++------------------- distribution/packages.bzl | 4 +-- 4 files changed, 29 insertions(+), 40 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index ea3658f978710..47b373d7602fb 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -698,12 +698,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Packaging rules for Bazel", project_desc = "Bazel rules for the packaging distributions", project_url = "https://github.com/bazelbuild/rules_pkg", - version = "ad57589abb069baa48f982778de408ea02d714fd", - sha256 = "ec14799a45f1d3b6c3e61c4d04513001bddac9208f09077b1f8c91ab47d234d2", - strip_prefix = "rules_pkg-{version}/pkg", + version = "e30c2506560d415f1c297b5e8a58b4758797e982", + sha256 = "a411945dacd9fe0e342aabcce6872a16d8efc2d68755177e1558c1a83774ed2b", + strip_prefix = "rules_pkg-{version}", urls = ["https://github.com/bazelbuild/rules_pkg/archive/{version}.tar.gz"], use_category = ["build"], - release_date = "2021-10-22", + release_date = "2021-12-10", ), org_llvm_llvm = dict( # When changing this, you must re-generate the list of llvm libs diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 91b11ad472f5c..1ed32b41cca37 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -233,9 +233,7 @@ elif [[ "$CI_TARGET" == "bazel.distribution" ]]; then "--action_env=PACKAGES_MAINTAINER_EMAIL") fi - # TODO(phlax): remove MakeDeb once issue with changes file is resolved BAZEL_BUILD_OPTIONS+=( - "--strategy=MakeDeb=sandboxed,local" "--strategy=Genrule=sandboxed,local" "--strategy=PackageTar=sandboxed,local") diff --git a/distribution/debian/packages.bzl b/distribution/debian/packages.bzl index 5b96dd6df2739..bfd5103642511 100644 --- a/distribution/debian/packages.bzl +++ b/distribution/debian/packages.bzl @@ -1,4 +1,5 @@ -load("@rules_pkg//:pkg.bzl", "pkg_deb", "pkg_tar") +load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") +load("@rules_pkg//pkg:deb.bzl", "pkg_deb") GLIBC_MIN_VERSION = "2.27" @@ -21,7 +22,7 @@ def envoy_pkg_deb( **kwargs): """Wrapper for `pkg_deb` with Envoy defaults""" pkg_deb( - name = "%s.deb" % name, + name = "%s-deb" % name, architecture = architecture, data = data, depends = depends, @@ -35,6 +36,17 @@ def envoy_pkg_deb( **kwargs ) + native.filegroup( + name = "%s.changes" % name, + srcs = ["%s-deb" % name], + output_group = "changes", + ) + native.filegroup( + name = "%s.deb" % name, + srcs = ["%s-deb" % name], + output_group = "deb", + ) + def envoy_pkg_debs(name, version, release_version, maintainer, bin_files = ":envoy-bin-files", config = ":envoy-config"): """Package the Envoy .debs with their .changes files. @@ -75,35 +87,14 @@ def envoy_pkg_debs(name, version, release_version, maintainer, bin_files = ":env maintainer = maintainer, ) - # TODO(phlax): Remove this hack - # Due to upstream issues with `OutputGroupInfo` files from `pkg_deb` - # we have to follow the real filepath of the .deb file to find the - # .changes file (~artefact) - # For this hack to work, strategy needs to be `sandbox,local` for the - # following mnemonics: - # - Genrule - # - MakeDeb - # - # Upstream issue is here: https://github.com/bazelbuild/rules_pkg/issues/477 - # - - deb_files = ( - "envoy_deb1=$$(realpath $(location :envoy.deb)) " + - "&& envoy_deb2=$$(realpath $(location :envoy-%s.deb)) \\" % release_version - ) - - # bundle all debs and changes files into /debs folder of tarball - native.genrule( + pkg_tar( name = name, - srcs = ["envoy.deb", "envoy-%s.deb" % release_version], - outs = [":debs.tar"], - cmd = deb_files + """ - && tar --transform "flags=r;s|^|deb/|" \ - -C $$(dirname $$envoy_deb1) \ - -cf $@ \ - $$(basename $${envoy_deb1}) \ - $$(basename $${envoy_deb1%.deb}.changes) \ - $$(basename $${envoy_deb2}) \ - $$(basename $${envoy_deb2%.deb}.changes) - """, + srcs = [ + "envoy.changes", + "envoy.deb", + "envoy-%s.changes" % release_version, + "envoy-%s.deb" % release_version, + ], + extension = "tar", + package_dir = "deb", ) diff --git a/distribution/packages.bzl b/distribution/packages.bzl index f293863bb755d..187585709c82b 100644 --- a/distribution/packages.bzl +++ b/distribution/packages.bzl @@ -1,5 +1,5 @@ -load("@rules_pkg//:pkg.bzl", "pkg_tar") -load("@rules_pkg//:mappings.bzl", "pkg_attributes", "pkg_files") +load("@rules_pkg//pkg:pkg.bzl", "pkg_tar") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files") load("//distribution/debian:packages.bzl", "envoy_pkg_debs") def _release_version_for(version): From 0f915b7ceddfeb302a2e6613e69274b4ac8d75c8 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 17 Dec 2021 13:04:40 +0000 Subject: [PATCH 05/11] cleanups-and-postinst Signed-off-by: Ryan Northey --- ci/do_ci.sh | 4 --- distribution/debian/BUILD | 1 + distribution/debian/packages.bzl | 2 ++ distribution/debian/postinst | 42 ++++++++++++++++++++++++++++++++ distribution/debian/preinst | 18 -------------- 5 files changed, 45 insertions(+), 22 deletions(-) create mode 100644 distribution/debian/postinst diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 1ed32b41cca37..888f8bf6d47ec 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -233,10 +233,6 @@ elif [[ "$CI_TARGET" == "bazel.distribution" ]]; then "--action_env=PACKAGES_MAINTAINER_EMAIL") fi - BAZEL_BUILD_OPTIONS+=( - "--strategy=Genrule=sandboxed,local" - "--strategy=PackageTar=sandboxed,local") - bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c opt //distribution:packages.tar.gz if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then cp -a bazel-bin/distribution/packages.tar.gz "${ENVOY_BUILD_DIR}/packages.x64.tar.gz" diff --git a/distribution/debian/BUILD b/distribution/debian/BUILD index bbcd2da9a7076..7aa9b46b84345 100644 --- a/distribution/debian/BUILD +++ b/distribution/debian/BUILD @@ -7,4 +7,5 @@ envoy_package() exports_files([ "copyright", "preinst", + "postinst", ]) diff --git a/distribution/debian/packages.bzl b/distribution/debian/packages.bzl index bfd5103642511..9848ad796bfe5 100644 --- a/distribution/debian/packages.bzl +++ b/distribution/debian/packages.bzl @@ -9,6 +9,7 @@ def envoy_pkg_deb( homepage = "https://www.envoyproxy.io/", description = "Envoy built for Debian/Ubuntu", preinst = "//distribution/debian:preinst", + postinst = "//distribution/debian:postinst", supported_distributions = "buster bullseye bionic focal hirstute impish", architecture = select({ "//bazel:x86": "amd64", @@ -33,6 +34,7 @@ def envoy_pkg_deb( package = name, version = version, preinst = preinst, + postinst = postinst, **kwargs ) diff --git a/distribution/debian/postinst b/distribution/debian/postinst new file mode 100644 index 0000000000000..d56003e6a9cd0 --- /dev/null +++ b/distribution/debian/postinst @@ -0,0 +1,42 @@ +#!/bin/sh + +# postinst script for envoy + +set -e + +case "$1" in + + configure) + cat <&2 + exit 1 + ;; + +esac + +#DEBHELPER# + +exit 0 diff --git a/distribution/debian/preinst b/distribution/debian/preinst index 6e67a52313bac..71de976079df8 100755 --- a/distribution/debian/preinst +++ b/distribution/debian/preinst @@ -23,28 +23,10 @@ addenvoyuser() { fi } - case "$1" in install) addenvoyuser - cat < Date: Fri, 17 Dec 2021 15:28:40 +0000 Subject: [PATCH 06/11] verification-script Signed-off-by: Ryan Northey --- distribution/BUILD | 9 +++++++++ distribution/verification.sh | 17 ----------------- 2 files changed, 9 insertions(+), 17 deletions(-) delete mode 100755 distribution/verification.sh diff --git a/distribution/BUILD b/distribution/BUILD index 7d1279b902c20..0001d7e1cb23c 100644 --- a/distribution/BUILD +++ b/distribution/BUILD @@ -21,6 +21,15 @@ envoy_pkg_distros( version = VERSION, ) +genrule( + name = "verification", + outs = ["verification.sh"], + cmd = """ + echo 'exec $${@}' > $@ \ + && chmod +x $@ + """, +) + sh_binary( name = "verify_packages", srcs = [":verification.sh"], diff --git a/distribution/verification.sh b/distribution/verification.sh deleted file mode 100755 index 5220268f0c662..0000000000000 --- a/distribution/verification.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -e - -echo "$@" - -VERIFY_COMMAND="$1" -TESTFILE="$2" -VERSION="$3" -CONFIG="$4" - -shift 4 - -echo "VERIFY_COMMAND: ${VERIFY_COMMAND}" -echo "TESTFILE: ${TESTFILE}" -echo "VERSION: ${VERSION}" -echo "CONFIG: ${CONFIG}" - -"$VERIFY_COMMAND" "$TESTFILE" "$VERSION" "$CONFIG" "${@}" From 12a4876444d9329e43930f2e5a2a2e4a75abdcf9 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Mon, 31 Jan 2022 09:04:38 +0000 Subject: [PATCH 07/11] bazel/ Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 6099e0ddb05b9..5f1bb7cb93528 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -680,12 +680,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Packaging rules for Bazel", project_desc = "Bazel rules for the packaging distributions", project_url = "https://github.com/bazelbuild/rules_pkg", - version = "e30c2506560d415f1c297b5e8a58b4758797e982", - sha256 = "a411945dacd9fe0e342aabcce6872a16d8efc2d68755177e1558c1a83774ed2b", + version = "62b1443f4f92cb2bfd81c47ed155e86cc6854786", + sha256 = "f9b0592eb1d4c0c7505f00f457a3e913fd483cc08bcca4c47cc7080c0894684a", strip_prefix = "rules_pkg-{version}", urls = ["https://github.com/bazelbuild/rules_pkg/archive/{version}.tar.gz"], use_category = ["build"], - release_date = "2021-12-10", + release_date = "2022-01-29", ), org_llvm_llvm = dict( # When changing this, you must re-generate the list of llvm libs From 8b003b2070a751350b7608e2de46a38409aeb377 Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Fri, 18 Feb 2022 14:51:41 +0000 Subject: [PATCH 08/11] remove-hirstute Signed-off-by: Ryan Northey --- distribution/debian/packages.bzl | 2 +- distribution/distros.yaml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/distribution/debian/packages.bzl b/distribution/debian/packages.bzl index 9848ad796bfe5..24d9281acad68 100644 --- a/distribution/debian/packages.bzl +++ b/distribution/debian/packages.bzl @@ -10,7 +10,7 @@ def envoy_pkg_deb( description = "Envoy built for Debian/Ubuntu", preinst = "//distribution/debian:preinst", postinst = "//distribution/debian:postinst", - supported_distributions = "buster bullseye bionic focal hirstute impish", + supported_distributions = "buster bullseye bionic focal impish", architecture = select({ "//bazel:x86": "amd64", "//conditions:default": "arm64", diff --git a/distribution/distros.yaml b/distribution/distros.yaml index c02180d91662a..549d4249dc652 100644 --- a/distribution/distros.yaml +++ b/distribution/distros.yaml @@ -15,10 +15,6 @@ ubuntu_focal: image: ubuntu:20.04 ext: focal.changes -ubuntu_hirstute: - image: ubuntu:21.04 - ext: hirstute.changes - ubuntu_impish: image: ubuntu:21.10 ext: impish.changes From d6592d7b2743f98141b31be94f6b7dcf60063c0d Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 17 Mar 2022 08:29:24 +0000 Subject: [PATCH 09/11] repo: Update build image and toolchains Signed-off-by: Ryan Northey --- .bazelrc | 2 +- .devcontainer/Dockerfile | 2 +- bazel/repository_locations.bzl | 6 +++--- examples/wasm-cc/docker-compose-wasm.yaml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.bazelrc b/.bazelrc index 80eef8d7e09f1..b17f12b3ee270 100644 --- a/.bazelrc +++ b/.bazelrc @@ -277,7 +277,7 @@ build:remote-clang-cl --config=rbe-toolchain-clang-cl # Docker sandbox # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/main/toolchains/rbe_toolchains_config.bzl#L8 -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:d859a503314ae611bb7ca4a7b4b4a19194e199f0 +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build-ubuntu:142e6d5662b98277a84c327da26ed266ab0e3191 build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a8e8f3a49cbee..27cc537dff092 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM gcr.io/envoy-ci/envoy-build:d859a503314ae611bb7ca4a7b4b4a19194e199f0 +FROM gcr.io/envoy-ci/envoy-build:142e6d5662b98277a84c327da26ed266ab0e3191 ARG USERNAME=vscode ARG USER_UID=501 diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index aa62957829d6b..91500a5ca618e 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -64,11 +64,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "envoy-build-tools", project_desc = "Common build tools shared by the Envoy/UDPA ecosystem", project_url = "https://github.com/envoyproxy/envoy-build-tools", - version = "f3a70cf47bd91d8f6ab080e4da361cc7f8e6f24d", - sha256 = "f18224874887fae1f897fa582fb7aad34fc7604cadb1d3fe92527eabdb8af513", + version = "f710be3099b65a2f260a632f8336a2c18e8324b9", + sha256 = "c7064405ab9dc4e04343147fc5fd399e90d25ccb849ede0d3e288b2e54c04634", strip_prefix = "envoy-build-tools-{version}", urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/{version}.tar.gz"], - release_date = "2022-03-11", + release_date = "2022-03-17", use_category = ["build"], ), boringssl = dict( diff --git a/examples/wasm-cc/docker-compose-wasm.yaml b/examples/wasm-cc/docker-compose-wasm.yaml index f89002cf7d148..b7214da955f70 100644 --- a/examples/wasm-cc/docker-compose-wasm.yaml +++ b/examples/wasm-cc/docker-compose-wasm.yaml @@ -2,7 +2,7 @@ version: "3.7" services: wasm_compile_update: - image: envoyproxy/envoy-build-ubuntu:d859a503314ae611bb7ca4a7b4b4a19194e199f0 + image: envoyproxy/envoy-build-ubuntu:142e6d5662b98277a84c327da26ed266ab0e3191 command: | bash -c "bazel build //examples/wasm-cc:envoy_filter_http_wasm_updated_example.wasm && cp -a bazel-bin/examples/wasm-cc/* /build" working_dir: /source @@ -11,7 +11,7 @@ services: - ./lib:/build wasm_compile: - image: envoyproxy/envoy-build-ubuntu:d859a503314ae611bb7ca4a7b4b4a19194e199f0 + image: envoyproxy/envoy-build-ubuntu:142e6d5662b98277a84c327da26ed266ab0e3191 command: | bash -c "bazel build //examples/wasm-cc:envoy_filter_http_wasm_example.wasm && cp -a bazel-bin/examples/wasm-cc/* /build" working_dir: /source From 53768cf8aee98665a18077629c4dd9b44af0c97c Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 17 Mar 2022 08:29:24 +0000 Subject: [PATCH 10/11] repo: Update build image and toolchains Signed-off-by: Ryan Northey --- bazel/repository_locations.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 91500a5ca618e..73efaef908d0c 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -68,7 +68,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( sha256 = "c7064405ab9dc4e04343147fc5fd399e90d25ccb849ede0d3e288b2e54c04634", strip_prefix = "envoy-build-tools-{version}", urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/{version}.tar.gz"], - release_date = "2022-03-17", + release_date = "2022-03-16", use_category = ["build"], ), boringssl = dict( From 776f9416c4ac8974702052405533c36a7f9bf57e Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 7 Apr 2022 15:06:06 +0100 Subject: [PATCH 11/11] improve-ci-key-name Signed-off-by: Ryan Northey --- ci/do_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 717a59e1e2de3..8cce3c15cde49 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -239,7 +239,7 @@ elif [[ "$CI_TARGET" == "bazel.distribution" ]]; then # and the packages signed with it, for the purpose of testing only. if ! gpg --list-secret-keys "*"; then export PACKAGES_MAINTAINER_NAME="Envoy CI" - export PACKAGES_MAINTAINER_EMAIL="envoy@snake.oil" + export PACKAGES_MAINTAINER_EMAIL="envoy-ci@for.testing.only" BAZEL_BUILD_OPTIONS+=( "--action_env=PACKAGES_GEN_KEY=1" "--action_env=PACKAGES_MAINTAINER_NAME"