From fc3097e19c1d3a7a68a275d91fb7993948114adc Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Wed, 28 Jul 2021 16:47:37 +0000 Subject: [PATCH 1/4] build: setup contrib Also move Squash extension to contrib Signed-off-by: Matt Klein --- .azure-pipelines/pipelines.yml | 16 ++++ .dockerignore | 4 +- BUILD | 7 ++ CONTRIBUTING.md | 10 +- EXTENSION_POLICY.md | 9 ++ REPO_LAYOUT.md | 12 ++- api/BUILD | 2 +- .../extensions/filters/http/squash/v3/BUILD | 0 .../filters/http/squash/v3/squash.proto | 0 api/test/validate/BUILD | 1 - api/test/validate/pgv_test.cc | 1 - api/versioning/BUILD | 2 +- bazel/README.md | 10 +- bazel/api_binding.bzl | 1 + bazel/envoy_build_system.bzl | 9 +- bazel/envoy_library.bzl | 8 ++ ci/Dockerfile-envoy | 5 +- ci/Dockerfile-envoy-alpine | 2 +- ci/Dockerfile-envoy-distroless | 2 +- ci/build_setup.sh | 2 + ci/do_ci.sh | 94 +++++++++++-------- ci/docker_ci.sh | 10 +- contrib/BUILD | 6 ++ contrib/all_contrib_extensions.bzl | 4 + contrib/contrib_build_config.bzl | 8 ++ contrib/exe/BUILD | 24 +++++ contrib/extensions_metadata.yaml | 5 + .../filters/http/squash/source}/BUILD | 14 +-- .../filters/http/squash/source}/config.cc | 9 +- .../filters/http/squash/source}/config.h | 6 +- .../http/squash/source}/squash_filter.cc | 4 +- .../http/squash/source}/squash_filter.h | 2 +- .../filters/http/squash/test}/BUILD | 28 +++--- .../filters/http/squash/test}/config_test.cc | 8 +- .../test}/squash_filter_integration_test.cc | 0 .../http/squash/test}/squash_filter_test.cc | 6 +- docs/BUILD | 8 +- docs/root/start/install.rst | 38 ++++++++ docs/root/version_history/current.rst | 4 + generated_api_shadow/BUILD | 2 +- .../extensions/filters/http/squash/v3/BUILD | 0 .../filters/http/squash/v3/squash.proto | 0 source/extensions/extensions_build_config.bzl | 1 - source/extensions/extensions_metadata.yaml | 5 - .../extensions/filters/http/common/fuzz/BUILD | 1 - .../http/common/fuzz/uber_per_filter.cc | 14 --- tools/code_format/check_format.py | 5 +- tools/code_format/envoy_build_fixer.py | 7 +- tools/docs/BUILD | 1 - tools/docs/generate_api_rst.py | 33 ++++--- .../docs/generate_extensions_security_rst.py | 10 +- tools/extensions/BUILD | 2 + tools/extensions/extensions_check.py | 46 +++++---- .../extensions/tests/test_extensions_check.py | 10 +- tools/gen_compilation_database.py | 9 +- tools/proto_format/proto_sync.py | 10 +- tools/protodoc/BUILD | 1 + tools/protodoc/protodoc.py | 55 ++++++++--- .../type_whisperer/proto_build_targets_gen.py | 5 + 59 files changed, 422 insertions(+), 176 deletions(-) rename api/{ => contrib}/envoy/extensions/filters/http/squash/v3/BUILD (100%) rename api/{ => contrib}/envoy/extensions/filters/http/squash/v3/squash.proto (100%) create mode 100644 contrib/BUILD create mode 100644 contrib/all_contrib_extensions.bzl create mode 100644 contrib/contrib_build_config.bzl create mode 100644 contrib/exe/BUILD create mode 100644 contrib/extensions_metadata.yaml rename {source/extensions/filters/http/squash => contrib/filters/http/squash/source}/BUILD (78%) rename {source/extensions/filters/http/squash => contrib/filters/http/squash/source}/config.cc (80%) rename {source/extensions/filters/http/squash => contrib/filters/http/squash/source}/config.h (84%) rename {source/extensions/filters/http/squash => contrib/filters/http/squash/source}/squash_filter.cc (98%) rename {source/extensions/filters/http/squash => contrib/filters/http/squash/source}/squash_filter.h (98%) rename {test/extensions/filters/http/squash => contrib/filters/http/squash/test}/BUILD (56%) rename {test/extensions/filters/http/squash => contrib/filters/http/squash/test}/config_test.cc (88%) rename {test/extensions/filters/http/squash => contrib/filters/http/squash/test}/squash_filter_integration_test.cc (100%) rename {test/extensions/filters/http/squash => contrib/filters/http/squash/test}/squash_filter_test.cc (99%) rename generated_api_shadow/{ => contrib}/envoy/extensions/filters/http/squash/v3/BUILD (100%) rename generated_api_shadow/{ => contrib}/envoy/extensions/filters/http/squash/v3/squash.proto (100%) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 527305ef41d83..e5f2904cfb54f 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -310,6 +310,13 @@ stages: itemPattern: "bazel.release/envoy_binary.tar.gz" downloadType: single targetPath: $(Build.StagingDirectory) + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release" + itemPattern: "bazel.release/envoy-contrib_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) - task: DownloadBuildArtifacts@0 inputs: buildType: current @@ -317,10 +324,19 @@ stages: itemPattern: "bazel.release.arm64/envoy_binary.tar.gz" downloadType: single targetPath: $(Build.StagingDirectory) + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release.arm64" + itemPattern: "bazel.release.arm64/envoy-contrib_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) - bash: | set -e mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64 + tar zxf $(Build.StagingDirectory)/bazel.release/envoy-contrib_binary.tar.gz -C ./linux/amd64 mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64 + tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy-contrib_binary.tar.gz -C ./linux/arm64 ci/docker_ci.sh workingDirectory: $(Build.SourcesDirectory) env: diff --git a/.dockerignore b/.dockerignore index ee59c60405f33..e0ef1fd120316 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,8 +3,8 @@ !/ci !/configs/google-vrp !/configs/*yaml -!/linux/amd64/build_release* -!/linux/arm64/build_release* +!/linux/amd64/build_*release* +!/linux/arm64/build_*release* !/local !/test/config/integration/certs !/windows diff --git a/BUILD b/BUILD index 7ef982f5621bc..9e35562c085fb 100644 --- a/BUILD +++ b/BUILD @@ -33,3 +33,10 @@ package_group( "//test/extensions/...", ], ) + +package_group( + name = "contrib_library", + packages = [ + "//contrib/...", + ], +) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38cded6eaf8b5..7098c2ad46aa9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -262,7 +262,15 @@ Other changes will likely include * Editing [source/extensions/extensions_metadata.yaml](source/extensions/extensions_metadata.yaml) to include metadata for the new extensions * Editing [docs/root/api-v3/config/config.rst](docs/root/api-v3/config/config.rst) to add area/area * Adding `docs/root/api-v3/config/area/area.rst` to add a table of contents for the API docs - * Adding `source/extensions/area/well_known_names.h` for registered plugins + +# Adding contrib extensions + +See [EXTENSION_POLICY.md](EXTENSION_POLICY.md) for more information on contrib. Adding a contrib +extension mostly mirrors adding a normal extension above. Some differences are noted here: + +* API files should be added in `api/contrib/envoy/` +* Build config and metadata should be included in [contrib/contrib_build_config.bzl](contrib/contrib_build_config.bzl) + and [contrib/extensions_metadata.yaml](contrib/extensions_metadata.yaml). # DCO: Sign your work diff --git a/EXTENSION_POLICY.md b/EXTENSION_POLICY.md index 39d41f14f875f..3fed033ada676 100644 --- a/EXTENSION_POLICY.md +++ b/EXTENSION_POLICY.md @@ -137,3 +137,12 @@ case we need to install an extension point, which can be done as follows: 3. Update [extending envoy](docs/root/extending/extending.rst) to list the new extension point and add any documentation explaining the extension point. At the very least this should link to the corresponding proto. + +## Contrib extensions + +As described in [this document](https://docs.google.com/document/d/1yl7GOZK1TDm_7vxQvt8UQEAu07UQFru1uEKXM6ZZg_g/edit#), +Envoy allows an alternate path to adding extensions called "contrib." The barrier to entry for a +contrib extension is lower than a core extension, with the tradeoff that contrib extensions are not +included by default in the main image builds. Consumers need to pull directly from the contrib +images described in the installation guide. Please read the linked document in detail to determine +whether contrib extensions are the right choice for a newly proposed extension. diff --git a/REPO_LAYOUT.md b/REPO_LAYOUT.md index c29c0b518c3f7..070c3d6683753 100644 --- a/REPO_LAYOUT.md +++ b/REPO_LAYOUT.md @@ -66,7 +66,6 @@ Not every directory within test is described below, but a few highlights: We maintain a very specific code and namespace layout for extensions. This aids in discovering code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNERS). - * All extensions are either registered in [all_extensions.bzl](source/extensions/all_extensions.bzl) or [extensions_build_config.bzl](source/extensions/extensions_build_config.bzl). The former is for extensions that cannot be removed from the primary Envoy build. The latter is for extensions @@ -124,3 +123,14 @@ code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNE code that is used by both HTTP and network filters. Common code used only by two HTTP filters would be found in `filters/http/common/`. Common code should be placed in a common namespace. E.g., `Envoy::Extensions::Filters::Common`. + +## [contrib](contrib/) layout + +This directory contains contrib extensions. See [EXTENSION_POLICY.md](EXTENSION_POLICY.md) for +more information. + +* [contrib/exe/](contrib/exe/): The default executable for contrib. This is similar to the + `envoy-static` target but also includes all contrib extensions, and is used to produce the + contrib image targets. +* [contrib/...](contrib/): The rest of this directory mirrors the [source/extensions](source/extensions/) + layout. Contrib extensions are placed here. diff --git a/api/BUILD b/api/BUILD index 24e0ea0de018a..044bc7d137a88 100644 --- a/api/BUILD +++ b/api/BUILD @@ -57,6 +57,7 @@ proto_library( name = "v3_protos", visibility = ["//visibility:public"], deps = [ + "//contrib/envoy/extensions/filters/http/squash/v3:pkg", "//envoy/admin/v3:pkg", "//envoy/config/accesslog/v3:pkg", "//envoy/config/bootstrap/v3:pkg", @@ -143,7 +144,6 @@ proto_library( "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", "//envoy/extensions/filters/http/set_metadata/v3:pkg", - "//envoy/extensions/filters/http/squash/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", "//envoy/extensions/filters/http/wasm/v3:pkg", "//envoy/extensions/filters/listener/http_inspector/v3:pkg", diff --git a/api/envoy/extensions/filters/http/squash/v3/BUILD b/api/contrib/envoy/extensions/filters/http/squash/v3/BUILD similarity index 100% rename from api/envoy/extensions/filters/http/squash/v3/BUILD rename to api/contrib/envoy/extensions/filters/http/squash/v3/BUILD diff --git a/api/envoy/extensions/filters/http/squash/v3/squash.proto b/api/contrib/envoy/extensions/filters/http/squash/v3/squash.proto similarity index 100% rename from api/envoy/extensions/filters/http/squash/v3/squash.proto rename to api/contrib/envoy/extensions/filters/http/squash/v3/squash.proto diff --git a/api/test/validate/BUILD b/api/test/validate/BUILD index 049780c2ec036..79b6d6d91da23 100644 --- a/api/test/validate/BUILD +++ b/api/test/validate/BUILD @@ -20,7 +20,6 @@ api_cc_test( "@envoy_api//envoy/extensions/filters/http/ip_tagging/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/lua/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/router/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/mongo_proxy/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/redis_proxy/v3:pkg_cc_proto", diff --git a/api/test/validate/pgv_test.cc b/api/test/validate/pgv_test.cc index feee839c6bfaf..75f7692ba3e69 100644 --- a/api/test/validate/pgv_test.cc +++ b/api/test/validate/pgv_test.cc @@ -19,7 +19,6 @@ #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.validate.h" #include "envoy/extensions/filters/http/lua/v3/lua.pb.validate.h" #include "envoy/extensions/filters/http/router/v3/router.pb.validate.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.validate.h" #include "envoy/extensions/filters/network/mongo_proxy/v3/mongo_proxy.pb.validate.h" #include "envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.pb.validate.h" diff --git a/api/versioning/BUILD b/api/versioning/BUILD index ef972635a9585..33e48b6488fae 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -9,6 +9,7 @@ proto_library( name = "active_protos", visibility = ["//visibility:public"], deps = [ + "//contrib/envoy/extensions/filters/http/squash/v3:pkg", "//envoy/admin/v3:pkg", "//envoy/config/accesslog/v3:pkg", "//envoy/config/bootstrap/v3:pkg", @@ -95,7 +96,6 @@ proto_library( "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", "//envoy/extensions/filters/http/set_metadata/v3:pkg", - "//envoy/extensions/filters/http/squash/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", "//envoy/extensions/filters/http/wasm/v3:pkg", "//envoy/extensions/filters/listener/http_inspector/v3:pkg", diff --git a/bazel/README.md b/bazel/README.md index b8c2b0a03f8df..b8328b770269b 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -668,7 +668,10 @@ The following optional features can be enabled on the Bazel build command-line: Envoy uses a modular build which allows extensions to be removed if they are not needed or desired. Extensions that can be removed are contained in -[extensions_build_config.bzl](../source/extensions/extensions_build_config.bzl). +[extensions_build_config.bzl](../source/extensions/extensions_build_config.bzl). Contrib build +extensions are contained in [contrib_build_config.bzl](../contrib/contrib_build_config.bzl). Note +that contrib extensions are only included by default when building the contrib executable and in +the default contrib images pushed to Docker Hub. The extensions disabled by default can be enabled by adding the following parameter to Bazel, for example to enable `envoy.filters.http.kill_request` extension, add `--//source/extensions/filters/http/kill_request:enabled`. @@ -719,6 +722,11 @@ local_repository( ... ``` +When performing custom builds, it is acceptable to include contrib extensions as well. This can +be done by including the desired Bazel paths from [contrib_build_config.bzl](../contrib/contrib_build_config.bzl) +into the overriden `extensions_build_config.bzl`. (There is no need to specifically perform +a contrib build to include a contrib extension.) + ## Extra extensions If you are building your own Envoy extensions or custom Envoy builds and encounter visibility diff --git a/bazel/api_binding.bzl b/bazel/api_binding.bzl index 550685f6299a4..56d7c95f08a8d 100644 --- a/bazel/api_binding.bzl +++ b/bazel/api_binding.bzl @@ -8,6 +8,7 @@ def _default_envoy_api_impl(ctx): "test", "tools", "versioning", + "contrib", ] for d in api_dirs: ctx.symlink(ctx.path(ctx.attr.envoy_root).dirname.get_child(ctx.attr.reldir).get_child(d), d) diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index a0463ab770d04..4d671ab9562fa 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -6,6 +6,7 @@ load(":envoy_internal.bzl", "envoy_external_dep_path") load( ":envoy_library.bzl", _envoy_basic_cc_library = "envoy_basic_cc_library", + _envoy_cc_contrib_extension = "envoy_cc_contrib_extension", _envoy_cc_extension = "envoy_cc_extension", _envoy_cc_library = "envoy_cc_library", _envoy_cc_linux_library = "envoy_cc_linux_library", @@ -50,8 +51,8 @@ load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") def envoy_package(): native.package(default_visibility = ["//visibility:public"]) -def envoy_extension_package(enabled_default = True): - native.package(default_visibility = EXTENSION_PACKAGE_VISIBILITY) +def envoy_extension_package(enabled_default = True, default_visibility = EXTENSION_PACKAGE_VISIBILITY): + native.package(default_visibility = default_visibility) bool_flag( name = "enabled", @@ -63,6 +64,9 @@ def envoy_extension_package(enabled_default = True): flag_values = {":enabled": "True"}, ) +def envoy_contrib_package(): + envoy_extension_package(default_visibility = ["//:contrib_library"]) + # A genrule variant that can output a directory. This is useful when doing things like # generating a fuzz corpus mechanically. def _envoy_directory_genrule_impl(ctx): @@ -220,6 +224,7 @@ envoy_cc_binary = _envoy_cc_binary # Library wrappers (from envoy_library.bzl) envoy_basic_cc_library = _envoy_basic_cc_library envoy_cc_extension = _envoy_cc_extension +envoy_cc_contrib_extension = _envoy_cc_contrib_extension envoy_cc_library = _envoy_cc_library envoy_cc_linux_library = _envoy_cc_linux_library envoy_cc_posix_library = _envoy_cc_posix_library diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index c29f35b160c0c..ac74d1be29c96 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -71,6 +71,14 @@ def envoy_cc_extension( visibility = visibility, ) +def envoy_cc_contrib_extension( + name, + tags = [], + extra_visibility = [], + visibility = ["//:contrib_library"], + **kwargs): + envoy_cc_extension(name, tags, extra_visibility, visibility, **kwargs) + # Envoy C++ library targets should be specified with this function. def envoy_cc_library( name, diff --git a/ci/Dockerfile-envoy b/ci/Dockerfile-envoy index 1c7711d540668..ac3cd3e74c12c 100644 --- a/ci/Dockerfile-envoy +++ b/ci/Dockerfile-envoy @@ -13,11 +13,12 @@ RUN apt-get update && apt-get upgrade -y \ RUN mkdir -p /etc/envoy +ARG ENVOY_BINARY=envoy ARG ENVOY_BINARY_SUFFIX=_stripped -ADD ${TARGETPLATFORM}/build_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ +ADD ${TARGETPLATFORM}/build_${ENVOY_BINARY}_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ ADD configs/envoyproxy_io_proxy.yaml /etc/envoy/envoy.yaml -ADD ${TARGETPLATFORM}/build_release/su-exec /usr/local/bin/ +ADD ${TARGETPLATFORM}/build_${ENVOY_BINARY}_release/su-exec /usr/local/bin/ RUN chown root:root /usr/local/bin/su-exec && adduser --group --system envoy EXPOSE 10000 diff --git a/ci/Dockerfile-envoy-alpine b/ci/Dockerfile-envoy-alpine index 18b5aba78f4f2..36a23f6e3a2aa 100644 --- a/ci/Dockerfile-envoy-alpine +++ b/ci/Dockerfile-envoy-alpine @@ -6,7 +6,7 @@ RUN apk add --no-cache shadow su-exec \ && addgroup -S envoy && adduser --no-create-home -S envoy -G envoy ARG ENVOY_BINARY_SUFFIX=_stripped -ADD linux/amd64/build_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ +ADD linux/amd64/build_envoy_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ EXPOSE 10000 diff --git a/ci/Dockerfile-envoy-distroless b/ci/Dockerfile-envoy-distroless index 630ccac08304e..d2647f7b38557 100644 --- a/ci/Dockerfile-envoy-distroless +++ b/ci/Dockerfile-envoy-distroless @@ -3,7 +3,7 @@ FROM gcr.io/distroless/base-debian10:nonroot ADD configs/envoyproxy_io_proxy.yaml /etc/envoy/envoy.yaml ARG ENVOY_BINARY_SUFFIX=_stripped -ADD linux/amd64/build_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ +ADD linux/amd64/build_envoy_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ EXPOSE 10000 diff --git a/ci/build_setup.sh b/ci/build_setup.sh index cedbda30a9e86..977c6477d7dc2 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -10,6 +10,8 @@ export PPROF_PATH=/thirdparty_build/bin/pprof [ -z "${ENVOY_SRCDIR}" ] && export ENVOY_SRCDIR=/source [ -z "${ENVOY_BUILD_TARGET}" ] && export ENVOY_BUILD_TARGET=//source/exe:envoy-static [ -z "${ENVOY_BUILD_DEBUG_INFORMATION}" ] && export ENVOY_BUILD_DEBUG_INFORMATION=//source/exe:envoy-static.dwp +[ -z "${ENVOY_CONTRIB_BUILD_TARGET}" ] && export ENVOY_CONTRIB_BUILD_TARGET=//contrib/exe:envoy-static +[ -z "${ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION}" ] && export ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION=//contrib/exe:envoy-static.dwp [ -z "${ENVOY_BUILD_ARCH}" ] && { ENVOY_BUILD_ARCH=$(uname -m) export ENVOY_BUILD_ARCH diff --git a/ci/do_ci.sh b/ci/do_ci.sh index b4a4ea3c2542c..f4196156be972 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -62,47 +62,38 @@ function bazel_with_collection() { run_process_test_result } -function cp_binary_for_outside_access() { - DELIVERY_LOCATION="$1" - cp -f \ - bazel-bin/"${ENVOY_BIN}" \ - "${ENVOY_DELIVERY_DIR}"/"${DELIVERY_LOCATION}" -} - -function cp_debug_info_for_outside_access() { - DELIVERY_LOCATION="$1" - cp -f \ - bazel-bin/"${ENVOY_BIN}".dwp \ - "${ENVOY_DELIVERY_DIR}"/"${DELIVERY_LOCATION}".dwp -} - - function cp_binary_for_image_build() { + local BINARY_TYPE="$1" + local COMPILE_TYPE="$2" + local EXE_NAME="$3" + # TODO(mattklein123): Replace this with caching and a different job which creates images. local BASE_TARGET_DIR="${ENVOY_SRCDIR}${BUILD_ARCH_DIR}" + local TARGET_DIR=build_"${EXE_NAME}"_"${BINARY_TYPE}" + local FINAL_DELIVERY_DIR="${ENVOY_DELIVERY_DIR}"/"${EXE_NAME}" + echo "Copying binary for image build..." - COMPILE_TYPE="$2" - mkdir -p "${BASE_TARGET_DIR}"/build_"$1" - cp -f "${ENVOY_DELIVERY_DIR}"/envoy "${BASE_TARGET_DIR}"/build_"$1" + mkdir -p "${BASE_TARGET_DIR}"/"${TARGET_DIR}" + cp -f "${FINAL_DELIVERY_DIR}"/envoy "${BASE_TARGET_DIR}"/"${TARGET_DIR}" # Copy the su-exec utility binary into the image - cp -f bazel-bin/external/com_github_ncopa_suexec/su-exec "${BASE_TARGET_DIR}"/build_"$1" + cp -f bazel-bin/external/com_github_ncopa_suexec/su-exec "${BASE_TARGET_DIR}"/"${TARGET_DIR}" if [[ "${COMPILE_TYPE}" == "dbg" || "${COMPILE_TYPE}" == "opt" ]]; then - cp -f "${ENVOY_DELIVERY_DIR}"/envoy.dwp "${BASE_TARGET_DIR}"/build_"$1" + cp -f "${FINAL_DELIVERY_DIR}"/envoy.dwp "${BASE_TARGET_DIR}"/"${TARGET_DIR}" fi - mkdir -p "${BASE_TARGET_DIR}"/build_"$1"_stripped - strip "${ENVOY_DELIVERY_DIR}"/envoy -o "${BASE_TARGET_DIR}"/build_"$1"_stripped/envoy + mkdir -p "${BASE_TARGET_DIR}"/"${TARGET_DIR}"_stripped + strip "${FINAL_DELIVERY_DIR}"/envoy -o "${BASE_TARGET_DIR}"/"${TARGET_DIR}"_stripped/envoy # Copy for azp which doesn't preserve permissions, creating a tar archive - tar czf "${ENVOY_BUILD_DIR}"/envoy_binary.tar.gz -C "${BASE_TARGET_DIR}" build_"$1" build_"$1"_stripped + tar czf "${ENVOY_BUILD_DIR}"/"${EXE_NAME}"_binary.tar.gz -C "${BASE_TARGET_DIR}" "${TARGET_DIR}" "${TARGET_DIR}"_stripped # Remove binaries to save space, only if BUILD_REASON exists (running in AZP) [[ -z "${BUILD_REASON}" ]] || \ - rm -rf "${BASE_TARGET_DIR}"/build_"$1" "${BASE_TARGET_DIR}"/build_"$1"_stripped "${ENVOY_DELIVERY_DIR}"/envoy{,.dwp} \ + rm -rf "${BASE_TARGET_DIR:?}"/"${TARGET_DIR}" "${BASE_TARGET_DIR:?}"/"${TARGET_DIR}"_stripped "${FINAL_DELIVERY_DIR:?}"/envoy{,.dwp} \ bazel-bin/"${ENVOY_BIN}"{,.dwp} } function bazel_binary_build() { - BINARY_TYPE="$1" + local BINARY_TYPE="$1" if [[ "${BINARY_TYPE}" == "release" ]]; then COMPILE_TYPE="opt" elif [[ "${BINARY_TYPE}" == "debug" ]]; then @@ -116,31 +107,44 @@ function bazel_binary_build() { COMPILE_TYPE="fastbuild" fi - echo "Building..." - ENVOY_BIN=$(echo "${ENVOY_BUILD_TARGET}" | sed -e 's#^@\([^/]*\)/#external/\1#;s#^//##;s#:#/#') + local BUILD_TARGET="$2" + local BUILD_DEBUG_INFORMATION="$3" + local EXE_NAME="$4" + local FINAL_DELIVERY_DIR="${ENVOY_DELIVERY_DIR}"/"${EXE_NAME}" + mkdir -p "${FINAL_DELIVERY_DIR}" + + echo "Building (type=${BINARY_TYPE} target=${BUILD_TARGET} debug=${BUILD_DEBUG_INFORMATION} name=${EXE_NAME})..." + ENVOY_BIN=$(echo "${BUILD_TARGET}" | sed -e 's#^@\([^/]*\)/#external/\1#;s#^//##;s#:#/#') # This is a workaround for https://github.com/bazelbuild/bazel/issues/11834 [[ -n "${ENVOY_RBE}" ]] && rm -rf bazel-bin/"${ENVOY_BIN}"* - bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c "${COMPILE_TYPE}" "${ENVOY_BUILD_TARGET}" ${CONFIG_ARGS} + bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c "${COMPILE_TYPE}" "${BUILD_TARGET}" ${CONFIG_ARGS} collect_build_profile "${BINARY_TYPE}"_build # Copy the built envoy binary somewhere that we can access outside of the # container. - cp_binary_for_outside_access envoy + cp -f bazel-bin/"${ENVOY_BIN}" "${FINAL_DELIVERY_DIR}"/envoy if [[ "${COMPILE_TYPE}" == "dbg" || "${COMPILE_TYPE}" == "opt" ]]; then # Generate dwp file for debugging since we used split DWARF to reduce binary # size - bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c "${COMPILE_TYPE}" "${ENVOY_BUILD_DEBUG_INFORMATION}" ${CONFIG_ARGS} + bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c "${COMPILE_TYPE}" "${BUILD_DEBUG_INFORMATION}" ${CONFIG_ARGS} # Copy the debug information - cp_debug_info_for_outside_access envoy + cp -f bazel-bin/"${ENVOY_BIN}".dwp "${FINAL_DELIVERY_DIR}"/envoy.dwp fi # Build su-exec utility bazel build external:su-exec - cp_binary_for_image_build "${BINARY_TYPE}" "${COMPILE_TYPE}" + cp_binary_for_image_build "${BINARY_TYPE}" "${COMPILE_TYPE}" "${EXE_NAME}" +} + +function bazel_envoy_binary_build() { + bazel_binary_build "$1" "${ENVOY_BUILD_TARGET}" "${ENVOY_BUILD_DEBUG_INFORMATION}" envoy +} +function bazel_contrib_binary_build() { + bazel_binary_build "$1" "${ENVOY_CONTRIB_BUILD_TARGET}" "${ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION}" envoy-contrib } function run_process_test_result() { @@ -184,6 +188,10 @@ if [[ $# -ge 1 ]]; then else # Coverage test will add QUICHE tests by itself. COVERAGE_TEST_TARGETS=("//test/...") + if [[ "$CI_TARGET" == "bazel.release" ]]; then + # We test contrib on release only. + COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "//contrib/...") + fi TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "@com_googlesource_quiche//:ci_tests") fi @@ -200,18 +208,22 @@ if [[ "$CI_TARGET" == "bazel.release" ]]; then echo "Testing ${TEST_TARGETS[*]} with options: ${BAZEL_BUILD_OPTIONS[*]}" bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c opt "${TEST_TARGETS[@]}" - echo "bazel release build with tests..." - bazel_binary_build release + echo "bazel release build..." + bazel_envoy_binary_build release + + echo "bazel contrib release build..." + bazel_contrib_binary_build release + exit 0 elif [[ "$CI_TARGET" == "bazel.release.server_only" ]]; then setup_clang_toolchain echo "bazel release build..." - bazel_binary_build release + bazel_envoy_binary_build release exit 0 elif [[ "$CI_TARGET" == "bazel.sizeopt.server_only" ]]; then setup_clang_toolchain echo "bazel size optimized build..." - bazel_binary_build sizeopt + bazel_envoy_binary_build sizeopt exit 0 elif [[ "$CI_TARGET" == "bazel.sizeopt" ]]; then setup_clang_toolchain @@ -219,7 +231,7 @@ elif [[ "$CI_TARGET" == "bazel.sizeopt" ]]; then bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --config=sizeopt "${TEST_TARGETS[@]}" echo "bazel size optimized build with tests..." - bazel_binary_build sizeopt + bazel_envoy_binary_build sizeopt exit 0 elif [[ "$CI_TARGET" == "bazel.gcc" ]]; then BAZEL_BUILD_OPTIONS+=("--test_env=HEAPCHECK=") @@ -229,7 +241,7 @@ elif [[ "$CI_TARGET" == "bazel.gcc" ]]; then bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild -- "${TEST_TARGETS[@]}" echo "bazel release build with gcc..." - bazel_binary_build fastbuild + bazel_envoy_binary_build fastbuild exit 0 elif [[ "$CI_TARGET" == "bazel.debug" ]]; then setup_clang_toolchain @@ -237,12 +249,12 @@ elif [[ "$CI_TARGET" == "bazel.debug" ]]; then bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" echo "bazel debug build with tests..." - bazel_binary_build debug + bazel_envoy_binary_build debug exit 0 elif [[ "$CI_TARGET" == "bazel.debug.server_only" ]]; then setup_clang_toolchain echo "bazel debug build..." - bazel_binary_build debug + bazel_envoy_binary_build debug exit 0 elif [[ "$CI_TARGET" == "bazel.asan" ]]; then setup_clang_toolchain @@ -295,7 +307,7 @@ elif [[ "$CI_TARGET" == "bazel.dev" ]]; then # This doesn't go into CI but is available for developer convenience. echo "bazel fastbuild build with tests..." echo "Building..." - bazel_binary_build fastbuild + bazel_envoy_binary_build fastbuild echo "Testing ${TEST_TARGETS[*]}" bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" diff --git a/ci/docker_ci.sh b/ci/docker_ci.sh index 5ccf2dbd0c614..7fb99271c63cc 100755 --- a/ci/docker_ci.sh +++ b/ci/docker_ci.sh @@ -23,6 +23,7 @@ config_env() { build_platforms() { TYPE=$1 FILE_SUFFIX="${TYPE/-debug/}" + FILE_SUFFIX="${FILE_SUFFIX/-contrib/}" if is_windows; then echo "windows/amd64" @@ -36,8 +37,13 @@ build_platforms() { build_args() { TYPE=$1 FILE_SUFFIX="${TYPE/-debug/}" + FILE_SUFFIX="${FILE_SUFFIX/-contrib/}" printf ' -f ci/Dockerfile-envoy%s' "${FILE_SUFFIX}" + if [[ "${TYPE}" == *-contrib* ]]; then + printf ' --build-arg ENVOY_BINARY=envoy-contrib' + fi + if [[ "${TYPE}" == *-debug ]]; then printf ' --build-arg ENVOY_BINARY_SUFFIX=' elif [[ "${TYPE}" == "-google-vrp" ]]; then @@ -124,7 +130,7 @@ if is_windows; then BUILD_COMMAND=("build") else # "-google-vrp" must come afer "" to ensure we rebuild the local base image dependency. - BUILD_TYPES=("" "-debug" "-alpine" "-distroless" "-google-vrp") + BUILD_TYPES=("" "-debug" "-contrib" "-contrib-debug" "-alpine" "-distroless" "-google-vrp") # Configure docker-buildx tools BUILD_COMMAND=("buildx" "build") @@ -141,7 +147,7 @@ for BUILD_TYPE in "${BUILD_TYPES[@]}"; do build_images "${BUILD_TYPE}" "$image_tag" if ! is_windows; then - if [[ "$BUILD_TYPE" == "" || "$BUILD_TYPE" == "-alpine" ]]; then + if [[ "$BUILD_TYPE" == "" || "$BUILD_TYPE" == "-contrib" || "$BUILD_TYPE" == "-alpine" ]]; then # verify_examples expects the base and alpine images, and for them to be named `-dev` dev_image="envoyproxy/envoy${BUILD_TYPE}-dev:latest" docker tag "$image_tag" "$dev_image" diff --git a/contrib/BUILD b/contrib/BUILD new file mode 100644 index 0000000000000..aa0691c6142a8 --- /dev/null +++ b/contrib/BUILD @@ -0,0 +1,6 @@ +licenses(["notice"]) # Apache 2 + +exports_files([ + "extensions_metadata.yaml", + "contrib_build_config.bzl", +]) diff --git a/contrib/all_contrib_extensions.bzl b/contrib/all_contrib_extensions.bzl new file mode 100644 index 0000000000000..5a450825fd033 --- /dev/null +++ b/contrib/all_contrib_extensions.bzl @@ -0,0 +1,4 @@ +load(":contrib_build_config.bzl", "CONTRIB_EXTENSIONS") + +def envoy_all_contrib_extensions(): + return [v + "_envoy_extension" for v in CONTRIB_EXTENSIONS.values()] diff --git a/contrib/contrib_build_config.bzl b/contrib/contrib_build_config.bzl new file mode 100644 index 0000000000000..ddf21dcb07329 --- /dev/null +++ b/contrib/contrib_build_config.bzl @@ -0,0 +1,8 @@ +# See bazel/README.md for details on how this system works. +CONTRIB_EXTENSIONS = { + # + # HTTP filters + # + + "envoy.filters.http.squash": "//contrib/filters/http/squash/source:config", +} diff --git a/contrib/exe/BUILD b/contrib/exe/BUILD new file mode 100644 index 0000000000000..778ceb7dc7b45 --- /dev/null +++ b/contrib/exe/BUILD @@ -0,0 +1,24 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_binary", + "envoy_contrib_package", +) +load( + "//contrib:all_contrib_extensions.bzl", + "envoy_all_contrib_extensions", +) + +licenses(["notice"]) # Apache 2 + +envoy_contrib_package() + +alias( + name = "envoy", + actual = ":envoy-static", +) + +envoy_cc_binary( + name = "envoy-static", + stamped = True, + deps = ["//source/exe:envoy_main_entry_lib"] + envoy_all_contrib_extensions(), +) diff --git a/contrib/extensions_metadata.yaml b/contrib/extensions_metadata.yaml new file mode 100644 index 0000000000000..d841fc7d5f737 --- /dev/null +++ b/contrib/extensions_metadata.yaml @@ -0,0 +1,5 @@ +envoy.filters.http.squash: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: stable diff --git a/source/extensions/filters/http/squash/BUILD b/contrib/filters/http/squash/source/BUILD similarity index 78% rename from source/extensions/filters/http/squash/BUILD rename to contrib/filters/http/squash/source/BUILD index 65831764d4c7e..897149d5137fc 100644 --- a/source/extensions/filters/http/squash/BUILD +++ b/contrib/filters/http/squash/source/BUILD @@ -1,8 +1,8 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_cc_extension", + "envoy_cc_contrib_extension", "envoy_cc_library", - "envoy_extension_package", + "envoy_contrib_package", ) licenses(["notice"]) # Apache 2 @@ -10,7 +10,7 @@ licenses(["notice"]) # Apache 2 # L7 HTTP filter that implements the Squash microservice debugger # Public docs: docs/root/configuration/http_filters/squash_filter.rst -envoy_extension_package() +envoy_contrib_package() envoy_cc_library( name = "squash_filter_lib", @@ -29,19 +29,19 @@ envoy_cc_library( "//source/common/http:utility_lib", "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) -envoy_cc_extension( +envoy_cc_contrib_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], deps = [ + ":squash_filter_lib", "//envoy/registry", "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", - "//source/extensions/filters/http/squash:squash_filter_lib", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/squash/config.cc b/contrib/filters/http/squash/source/config.cc similarity index 80% rename from source/extensions/filters/http/squash/config.cc rename to contrib/filters/http/squash/source/config.cc index d2420a6bb8381..7d65a5416ed66 100644 --- a/source/extensions/filters/http/squash/config.cc +++ b/contrib/filters/http/squash/source/config.cc @@ -1,12 +1,13 @@ -#include "source/extensions/filters/http/squash/config.h" +#include "contrib/filters/http/squash/source/config.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" #include "envoy/registry/registry.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" -#include "source/extensions/filters/http/squash/squash_filter.h" + +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" +#include "contrib/filters/http/squash/source/squash_filter.h" namespace Envoy { namespace Extensions { diff --git a/source/extensions/filters/http/squash/config.h b/contrib/filters/http/squash/source/config.h similarity index 84% rename from source/extensions/filters/http/squash/config.h rename to contrib/filters/http/squash/source/config.h index c1514fe61ab97..feff75eaf8b0e 100644 --- a/source/extensions/filters/http/squash/config.h +++ b/contrib/filters/http/squash/source/config.h @@ -1,10 +1,10 @@ #pragma once -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" - #include "source/extensions/filters/http/common/factory_base.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" + namespace Envoy { namespace Extensions { namespace HttpFilters { diff --git a/source/extensions/filters/http/squash/squash_filter.cc b/contrib/filters/http/squash/source/squash_filter.cc similarity index 98% rename from source/extensions/filters/http/squash/squash_filter.cc rename to contrib/filters/http/squash/source/squash_filter.cc index e2db20a9f2d2e..a906505bf0aca 100644 --- a/source/extensions/filters/http/squash/squash_filter.cc +++ b/contrib/filters/http/squash/source/squash_filter.cc @@ -1,8 +1,7 @@ -#include "source/extensions/filters/http/squash/squash_filter.h" +#include "contrib/filters/http/squash/source/squash_filter.h" #include -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" #include "envoy/http/codes.h" #include "source/common/common/empty_string.h" @@ -16,6 +15,7 @@ #include "source/common/protobuf/utility.h" #include "absl/container/fixed_array.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" namespace Envoy { namespace Extensions { diff --git a/source/extensions/filters/http/squash/squash_filter.h b/contrib/filters/http/squash/source/squash_filter.h similarity index 98% rename from source/extensions/filters/http/squash/squash_filter.h rename to contrib/filters/http/squash/source/squash_filter.h index 6786a6117a741..df1688210ac4b 100644 --- a/source/extensions/filters/http/squash/squash_filter.h +++ b/contrib/filters/http/squash/source/squash_filter.h @@ -2,7 +2,6 @@ #include -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" #include "envoy/http/async_client.h" #include "envoy/http/filter.h" #include "envoy/json/json_object.h" @@ -12,6 +11,7 @@ #include "source/common/protobuf/protobuf.h" #include "absl/types/optional.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/squash/BUILD b/contrib/filters/http/squash/test/BUILD similarity index 56% rename from test/extensions/filters/http/squash/BUILD rename to contrib/filters/http/squash/test/BUILD index f03de9e17e6d3..862ff5a09e720 100644 --- a/test/extensions/filters/http/squash/BUILD +++ b/contrib/filters/http/squash/test/BUILD @@ -1,52 +1,46 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_package", -) -load( - "//test/extensions:extensions_build_system.bzl", - "envoy_extension_cc_test", + "envoy_cc_test", + "envoy_contrib_package", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_contrib_package() -envoy_extension_cc_test( +envoy_cc_test( name = "squash_filter_test", srcs = ["squash_filter_test.cc"], - extension_names = ["envoy.filters.http.squash"], deps = [ + "//contrib/filters/http/squash/source:squash_filter_lib", "//envoy/event:dispatcher_interface", "//source/common/http:header_map_lib", "//source/common/stats:stats_lib", - "//source/extensions/filters/http/squash:squash_filter_lib", "//test/mocks/http:http_mocks", "//test/mocks/server:factory_context_mocks", "//test/mocks/upstream:cluster_manager_mocks", "//test/test_common:environment_lib", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) -envoy_extension_cc_test( +envoy_cc_test( name = "squash_filter_integration_test", srcs = ["squash_filter_integration_test.cc"], - extension_names = ["envoy.filters.http.squash"], deps = [ - "//source/extensions/filters/http/squash:config", + "//contrib/filters/http/squash/source:config", "//test/integration:http_integration_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", ], ) -envoy_extension_cc_test( +envoy_cc_test( name = "config_test", srcs = ["config_test.cc"], - extension_names = ["envoy.filters.http.squash"], deps = [ - "//source/extensions/filters/http/squash:config", + "//contrib/filters/http/squash/source:config", "//test/mocks/server:factory_context_mocks", "//test/test_common:utility_lib", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/filters/http/squash/config_test.cc b/contrib/filters/http/squash/test/config_test.cc similarity index 88% rename from test/extensions/filters/http/squash/config_test.cc rename to contrib/filters/http/squash/test/config_test.cc index 8964bb665308c..7561664845a26 100644 --- a/test/extensions/filters/http/squash/config_test.cc +++ b/contrib/filters/http/squash/test/config_test.cc @@ -1,11 +1,9 @@ -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" - -#include "source/extensions/filters/http/squash/config.h" - #include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" +#include "contrib/filters/http/squash/source/config.h" #include "gmock/gmock.h" #include "gtest/gtest.h" diff --git a/test/extensions/filters/http/squash/squash_filter_integration_test.cc b/contrib/filters/http/squash/test/squash_filter_integration_test.cc similarity index 100% rename from test/extensions/filters/http/squash/squash_filter_integration_test.cc rename to contrib/filters/http/squash/test/squash_filter_integration_test.cc diff --git a/test/extensions/filters/http/squash/squash_filter_test.cc b/contrib/filters/http/squash/test/squash_filter_test.cc similarity index 99% rename from test/extensions/filters/http/squash/squash_filter_test.cc rename to contrib/filters/http/squash/test/squash_filter_test.cc index be5d8d852bfe3..82ed3181e07c0 100644 --- a/test/extensions/filters/http/squash/squash_filter_test.cc +++ b/contrib/filters/http/squash/test/squash_filter_test.cc @@ -2,18 +2,16 @@ #include #include -#include "envoy/common/scope_tracker.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" - #include "source/common/http/message_impl.h" #include "source/common/protobuf/protobuf.h" -#include "source/extensions/filters/http/squash/squash_filter.h" #include "test/mocks/server/factory_context.h" #include "test/mocks/upstream/cluster_manager.h" #include "test/test_common/environment.h" #include "test/test_common/utility.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/filters/http/squash/source/squash_filter.h" #include "fmt/format.h" #include "gmock/gmock.h" #include "gtest/gtest.h" diff --git a/docs/BUILD b/docs/BUILD index e2a0c33e0114f..51f3e2b4a1cd3 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -115,11 +115,15 @@ pkg_tar( genrule( name = "extensions_security_rst", - srcs = ["//source/extensions:extensions_metadata.yaml"], + srcs = [ + "//source/extensions:extensions_metadata.yaml", + "//contrib:extensions_metadata.yaml", + ], outs = ["extensions_security_rst.tar"], cmd = """ $(location //tools/docs:generate_extensions_security_rst) \\ - $(location //source/extensions:extensions_metadata.yaml) $@ + $(location //source/extensions:extensions_metadata.yaml) \\ + $(location //contrib:extensions_metadata.yaml) $@ """, tools = ["//tools/docs:generate_extensions_security_rst"], ) diff --git a/docs/root/start/install.rst b/docs/root/start/install.rst index 79a651c12f3b5..30d6a17c03503 100644 --- a/docs/root/start/install.rst +++ b/docs/root/start/install.rst @@ -110,6 +110,20 @@ The following commands will pull and show the Envoy version of current images. $ docker pull envoyproxy/|envoy_distroless_docker_image| $ docker run --rm envoyproxy/|envoy_distroless_docker_image| --version +.. _install_contrib: + +Contrib images +~~~~~~~~~~~~~~ + +As described in `this document `_, +the Envoy project allows extensions to enter the repository as "contrib" extensions. The requirements +for such extensions are lower, and as such they are only available by default in special images. +The `envoyproxy/envoy-contrib `_ image +contains all contrib extensions on top of an Ubuntu base. The +`envoyproxy/envoy-contrib-debug `_ +image contains all contrib extensions on top of an Ubuntu base as well as debug symbols. Throughout +the documentation, extensions are clearly marked as being a contrib extension or a core extension. + .. _install_binaries: Pre-built Envoy Docker images @@ -140,6 +154,12 @@ The following table shows the available Docker images - |DOCKER_IMAGE_TAG_NAME| - - + * - `envoyproxy/envoy-contrib `_ + - Release :ref:`contrib ` binary with symbols stripped on top of an Ubuntu Bionic base. + - |DOCKER_IMAGE_TAG_NAME| + - |DOCKER_IMAGE_TAG_NAME| + - + - * - `envoyproxy/envoy-distroless `_ - Release binary with symbols stripped on top of a distroless base. - |DOCKER_IMAGE_TAG_NAME| @@ -164,12 +184,24 @@ The following table shows the available Docker images - |DOCKER_IMAGE_TAG_NAME| - - + * - `envoyproxy/envoy-contrib-debug `_ + - Release :ref:`contrib ` binary with debug symbols on top of an Ubuntu Bionic base. + - |DOCKER_IMAGE_TAG_NAME| + - |DOCKER_IMAGE_TAG_NAME| + - + - * - `envoyproxy/envoy-dev `_ - Release binary with symbols stripped on top of an Ubuntu Bionic base. - - - latest - latest + * - `envoyproxy/envoy-contrib-dev `_ + - Release :ref:`contrib ` binary with symbols stripped on top of an Ubuntu Bionic base. + - + - + - latest + - latest * - `envoyproxy/envoy-distroless-dev `_ - Release binary with symbols stripped on top of a distroless base. - @@ -188,6 +220,12 @@ The following table shows the available Docker images - - latest - latest + * - `envoyproxy/envoy-contrib-debug-dev `_ + - Release :ref:`contrib ` binary with debug symbols on top of an Ubuntu Bionic base. + - + - + - latest + - latest * - `envoyproxy/envoy-windows-dev `_ - Release binary with symbols stripped on top of a Windows Server 1809 base. Includes build tools. - diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 929a7038eac53..0b44d3d6f1546 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -5,6 +5,9 @@ Incompatible Behavior Changes ----------------------------- *Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* +* contrib: the :ref:`squash filter ` has been moved to + :ref:`contrib images `. + Minor Behavior Changes ---------------------- *Changes that may cause incompatibilities for some users, but should not for most* @@ -60,6 +63,7 @@ New Features ------------ * access_log: added :ref:`METADATA` token to handle all types of metadata (DYNAMIC, CLUSTER, ROUTE). * bootstrap: added :ref:`inline_headers ` in the bootstrap to make custom inline headers bootstrap configurable. +* contrib: added new :ref:`contrib images ` which contain contrib extensions. * http: added :ref:`string_match ` in the header matcher. * http: added support for :ref:`max_requests_per_connection ` for both upstream and downstream connections. * jwt_authn: added support for :ref:`Jwt Cache ` and its size can be specified by :ref:`jwt_cache_size `. diff --git a/generated_api_shadow/BUILD b/generated_api_shadow/BUILD index 24e0ea0de018a..044bc7d137a88 100644 --- a/generated_api_shadow/BUILD +++ b/generated_api_shadow/BUILD @@ -57,6 +57,7 @@ proto_library( name = "v3_protos", visibility = ["//visibility:public"], deps = [ + "//contrib/envoy/extensions/filters/http/squash/v3:pkg", "//envoy/admin/v3:pkg", "//envoy/config/accesslog/v3:pkg", "//envoy/config/bootstrap/v3:pkg", @@ -143,7 +144,6 @@ proto_library( "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", "//envoy/extensions/filters/http/set_metadata/v3:pkg", - "//envoy/extensions/filters/http/squash/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", "//envoy/extensions/filters/http/wasm/v3:pkg", "//envoy/extensions/filters/listener/http_inspector/v3:pkg", diff --git a/generated_api_shadow/envoy/extensions/filters/http/squash/v3/BUILD b/generated_api_shadow/contrib/envoy/extensions/filters/http/squash/v3/BUILD similarity index 100% rename from generated_api_shadow/envoy/extensions/filters/http/squash/v3/BUILD rename to generated_api_shadow/contrib/envoy/extensions/filters/http/squash/v3/BUILD diff --git a/generated_api_shadow/envoy/extensions/filters/http/squash/v3/squash.proto b/generated_api_shadow/contrib/envoy/extensions/filters/http/squash/v3/squash.proto similarity index 100% rename from generated_api_shadow/envoy/extensions/filters/http/squash/v3/squash.proto rename to generated_api_shadow/contrib/envoy/extensions/filters/http/squash/v3/squash.proto diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 8fbb6ffcee499..3f8dfeb00f0bc 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -103,7 +103,6 @@ EXTENSIONS = { "envoy.filters.http.rbac": "//source/extensions/filters/http/rbac:config", "envoy.filters.http.router": "//source/extensions/filters/http/router:config", "envoy.filters.http.set_metadata": "//source/extensions/filters/http/set_metadata:config", - "envoy.filters.http.squash": "//source/extensions/filters/http/squash:config", "envoy.filters.http.tap": "//source/extensions/filters/http/tap:config", "envoy.filters.http.wasm": "//source/extensions/filters/http/wasm:config", diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 44e8389f1eb9b..aaef9bda41909 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -263,11 +263,6 @@ envoy.filters.http.set_metadata: - envoy.filters.http security_posture: robust_to_untrusted_downstream_and_upstream status: stable -envoy.filters.http.squash: - categories: - - envoy.filters.http - security_posture: requires_trusted_downstream_and_upstream - status: stable envoy.filters.http.tap: categories: - envoy.filters.http diff --git a/test/extensions/filters/http/common/fuzz/BUILD b/test/extensions/filters/http/common/fuzz/BUILD index ad1be79cf3413..04bc4579103fe 100644 --- a/test/extensions/filters/http/common/fuzz/BUILD +++ b/test/extensions/filters/http/common/fuzz/BUILD @@ -57,7 +57,6 @@ envoy_cc_test_library( "//test/test_common:test_runtime_lib", "@envoy_api//envoy/extensions/filters/http/grpc_json_transcoder/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/jwt_authn/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/tap/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc index ce6720a2d2bf2..b1e2675f8b60c 100644 --- a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc @@ -1,6 +1,5 @@ #include "envoy/extensions/filters/http/grpc_json_transcoder/v3/transcoder.pb.h" #include "envoy/extensions/filters/http/jwt_authn/v3/config.pb.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" #include "envoy/extensions/filters/http/tap/v3/tap.pb.h" #include "source/common/tracing/http_tracer_impl.h" @@ -74,17 +73,6 @@ void UberFilterFuzzer::guideAnyProtoType(test::fuzz::HttpData* mutable_data, uin mutable_any->set_type_url(type_url); } -void cleanAttachmentTemplate(Protobuf::Message* message) { - envoy::extensions::filters::http::squash::v3::Squash& config = - dynamic_cast(*message); - std::string json; - Protobuf::util::JsonPrintOptions json_options; - if (!Protobuf::util::MessageToJsonString(config.attachment_template(), &json, json_options) - .ok()) { - config.clear_attachment_template(); - } -} - void cleanTapConfig(Protobuf::Message* message) { envoy::extensions::filters::http::tap::v3::Tap& config = dynamic_cast(*message); @@ -111,8 +99,6 @@ void UberFilterFuzzer::cleanFuzzedConfig(absl::string_view filter_name, if (filter_name == "envoy.filters.http.grpc_json_transcoder") { // Add a valid service proto descriptor. addBookstoreProtoDescriptor(message); - } else if (name == "envoy.filters.http.squash") { - cleanAttachmentTemplate(message); } else if (name == "envoy.filters.http.tap") { // TapDS oneof field and OutputSinkType StreamingGrpc not implemented cleanTapConfig(message); diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index cddeafcc42a3a..078715a96091c 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -23,6 +23,7 @@ "./bazel-", "./.cache", "./source/extensions/extensions_build_config.bzl", + "./contrib/contrib_build_config.bzl", "./bazel/toolchains/configs/", "./tools/testdata/check_format/", "./tools/pyformat/", @@ -97,8 +98,8 @@ "./source/common/common/regex.cc", "./source/common/stats/tag_extractor_impl.h", "./source/common/stats/tag_extractor_impl.cc", "./source/common/formatter/substitution_formatter.cc", - "./source/extensions/filters/http/squash/squash_filter.h", - "./source/extensions/filters/http/squash/squash_filter.cc", "./source/server/admin/utils.h", + "./contrib/filters/http/squash/source/squash_filter.h", + "./contrib/filters/http/squash/source/squash_filter.cc", "./source/server/admin/utils.h", "./source/server/admin/utils.cc", "./source/server/admin/stats_handler.h", "./source/server/admin/stats_handler.cc", "./source/server/admin/prometheus_stats.h", "./source/server/admin/prometheus_stats.cc", "./tools/clang_tools/api_booster/main.cc", diff --git a/tools/code_format/envoy_build_fixer.py b/tools/code_format/envoy_build_fixer.py index cfdbf59d24a90..7de2a307889c0 100755 --- a/tools/code_format/envoy_build_fixer.py +++ b/tools/code_format/envoy_build_fixer.py @@ -34,6 +34,7 @@ # Match a load() statement for the envoy_package macros. PACKAGE_LOAD_BLOCK_REGEX = re.compile('("envoy_package".*?\)\n)', re.DOTALL) EXTENSION_PACKAGE_LOAD_BLOCK_REGEX = re.compile('("envoy_extension_package".*?\)\n)', re.DOTALL) +CONTRIB_PACKAGE_LOAD_BLOCK_REGEX = re.compile('("envoy_contrib_package".*?\)\n)', re.DOTALL) # Match Buildozer 'print' output. Example of Buildozer print output: # cc_library json_transcoder_filter_lib [json_transcoder_filter.cc] (missing) (missing) @@ -41,7 +42,7 @@ '\s*([\w_]+)\s+([\w_]+)\s+[(\[](.*?)[)\]]\s+[(\[](.*?)[)\]]\s+[(\[](.*?)[)\]]') # Match API header include in Envoy source file? -API_INCLUDE_REGEX = re.compile('#include "(envoy/.*)/[^/]+\.pb\.(validate\.)?h"') +API_INCLUDE_REGEX = re.compile('#include "(contrib/envoy/.*|envoy/.*)/[^/]+\.pb\.(validate\.)?h"') class EnvoyBuildFixerError(Exception): @@ -79,6 +80,10 @@ def fix_package_and_license(path, contents): regex_to_use = EXTENSION_PACKAGE_LOAD_BLOCK_REGEX package_string = 'envoy_extension_package' + if 'contrib/' in path: + regex_to_use = CONTRIB_PACKAGE_LOAD_BLOCK_REGEX + package_string = 'envoy_contrib_package' + # Ensure we have an envoy_package import load if this is a real Envoy package. We also allow # the prefix to be overridden if envoy is included in a larger workspace. if re.search(ENVOY_RULE_REGEX, contents): diff --git a/tools/docs/BUILD b/tools/docs/BUILD index 39049acc40cfa..56892499d24d5 100644 --- a/tools/docs/BUILD +++ b/tools/docs/BUILD @@ -10,7 +10,6 @@ envoy_package() py_binary( name = "generate_extensions_security_rst", srcs = ["generate_extensions_security_rst.py"], - data = ["//source/extensions:extensions_metadata.yaml"], deps = [ "//tools/base:utils", ], diff --git a/tools/docs/generate_api_rst.py b/tools/docs/generate_api_rst.py index 2076b50d44277..e5539332de1fb 100644 --- a/tools/docs/generate_api_rst.py +++ b/tools/docs/generate_api_rst.py @@ -4,6 +4,24 @@ import tarfile +def include_package(envoy_api_protos, rst_file_path, prefix): + # `envoy_api_rst_files` is a list of file paths for .proto.rst files + # generated by protodoc + # + # we are only interested in the proto files generated for envoy protos, + # not for non-envoy dependencies + if ("pkg/" + prefix) not in rst_file_path: + return None + # derive the "canonical" path from the filepath + canonical = f"{rst_file_path.split('pkg/' + prefix)[1]}" + + # we are only interested in the actual v3 protos, not their dependencies + if (prefix + canonical) not in envoy_api_protos: + return None + + return canonical + + def main(): proto_srcs = sys.argv[1] envoy_api_rst_files = sys.argv[1:-1] @@ -24,19 +42,12 @@ def main(): ] for rst_file_path in envoy_api_rst_files: - # `envoy_api_rst_files` is a list of file paths for .proto.rst files - # generated by protodoc - # - # we are only interested in the proto files generated for envoy protos, - # not for non-envoy dependencies - if "pkg/envoy" not in rst_file_path: + canonical = include_package(envoy_api_protos, rst_file_path, "envoy/") + if canonical is None: + canonical = include_package(envoy_api_protos, rst_file_path, "contrib/envoy/") + if canonical is None: continue - # derive the "canonical" path from the filepath - canonical = f"{rst_file_path.split('pkg/envoy/')[1]}" - # we are only interested in the actual v3 protos, not their dependencies - if f"envoy/{canonical}" not in envoy_api_protos: - continue target = os.path.join("rst-out/api-v3", canonical) if not os.path.exists(os.path.dirname(target)): os.makedirs(os.path.dirname(target)) diff --git a/tools/docs/generate_extensions_security_rst.py b/tools/docs/generate_extensions_security_rst.py index 2dc6d8e84ee29..180e2eb247f69 100644 --- a/tools/docs/generate_extensions_security_rst.py +++ b/tools/docs/generate_extensions_security_rst.py @@ -18,16 +18,24 @@ def format_item(extension, metadata): item = '* :ref:`%s `' % (extension, extension) if metadata.get('status') == 'alpha': item += ' (alpha)' + if metadata.get('contrib', False): + item += ' (:ref:`contrib builds ` only)' return item def main(): metadata_filepath = sys.argv[1] - output_filename = sys.argv[2] + contrib_metadata_filepath = sys.argv[2] + output_filename = sys.argv[3] generated_rst_dir = os.path.dirname(output_filename) security_rst_root = os.path.join(generated_rst_dir, "intro/arch_overview/security") extension_db = utils.from_yaml(metadata_filepath) + contrib_extension_db = utils.from_yaml(contrib_metadata_filepath) + for contrib_extension in contrib_extension_db.keys(): + contrib_extension_db[contrib_extension]['contrib'] = True + extension_db.update(contrib_extension_db) + pathlib.Path(security_rst_root).mkdir(parents=True, exist_ok=True) security_postures = defaultdict(list) diff --git a/tools/extensions/BUILD b/tools/extensions/BUILD index b3f2dcbecaddb..52147a0e6b446 100644 --- a/tools/extensions/BUILD +++ b/tools/extensions/BUILD @@ -9,6 +9,8 @@ envoy_package() envoy_py_binary( name = "tools.extensions.extensions_check", data = [ + "//contrib:contrib_build_config.bzl", + "//contrib:extensions_metadata.yaml", "//source/extensions:extensions_metadata.yaml", "//source/extensions:extensions_build_config.bzl", "//test/extensions/filters/network/common/fuzz:uber_per_readfilter.cc", diff --git a/tools/extensions/extensions_check.py b/tools/extensions/extensions_check.py index 7a4b50fa52d83..8f67ad1e72a7d 100644 --- a/tools/extensions/extensions_check.py +++ b/tools/extensions/extensions_check.py @@ -14,6 +14,7 @@ from tools.base import checker, utils BUILD_CONFIG_PATH = "source/extensions/extensions_build_config.bzl" +CONTRIB_BUILD_CONFIG_PATH = "contrib/contrib_build_config.bzl" BUILTIN_EXTENSIONS = ( "envoy.request_id.uuid", "envoy.upstreams.tcp.generic", "envoy.transport_sockets.tls", @@ -70,6 +71,7 @@ FUZZ_TEST_PATH = "test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc" METADATA_PATH = "source/extensions/extensions_metadata.yaml" +CONTRIB_METADATA_PATH = "contrib/extensions_metadata.yaml" class ExtensionsConfigurationError(Exception): @@ -84,25 +86,18 @@ class ExtensionsChecker(checker.Checker): @cached_property def all_extensions(self) -> set: - return set(self.configured_extensions.keys()) | set(BUILTIN_EXTENSIONS) + return set(self.configured_extensions.keys()) | set( + self.configured_contrib_extensions.keys()) | set(BUILTIN_EXTENSIONS) @cached_property def configured_extensions(self) -> dict: - # source/extensions/extensions_build_config.bzl must have a - # .bzl suffix for Starlark import, so we are forced to do this workaround. - _extensions_build_config_spec = spec_from_loader( - "extensions_build_config", - SourceFileLoader("extensions_build_config", BUILD_CONFIG_PATH)) + return ExtensionsChecker._load_build_config( + "extensions_build_config", BUILD_CONFIG_PATH, "EXTENSIONS") - if not isinstance(_extensions_build_config_spec, ModuleSpec): - raise ExtensionsConfigurationError(f"Unable to parse build config {BUILD_CONFIG_PATH}") - extensions_build_config = module_from_spec(_extensions_build_config_spec) - - if not isinstance(_extensions_build_config_spec.loader, Loader): - raise ExtensionsConfigurationError(f"Unable to parse build config {BUILD_CONFIG_PATH}") - - _extensions_build_config_spec.loader.exec_module(extensions_build_config) - return getattr(extensions_build_config, "EXTENSIONS") + @cached_property + def configured_contrib_extensions(self) -> dict: + return ExtensionsChecker._load_build_config( + "contrib_build_config", CONTRIB_BUILD_CONFIG_PATH, "CONTRIB_EXTENSIONS") @property def fuzzed_count(self) -> int: @@ -114,8 +109,10 @@ def fuzzed_count(self) -> int: @cached_property def metadata(self) -> dict: result = utils.from_yaml(METADATA_PATH) + result.update(utils.from_yaml(CONTRIB_METADATA_PATH)) if not isinstance(result, dict): - raise ExtensionsConfigurationError(f"Unable to parse configuration: {METADATA_PATH}") + raise ExtensionsConfigurationError( + f"Unable to parse metadata: {METADATA_PATH} {CONTRIB_METADATA_PATH}") return result @property @@ -126,6 +123,23 @@ def robust_to_downstream_count(self) -> int: if "network" in ext and data["security_posture"] == "robust_to_untrusted_downstream" ]) + @staticmethod + def _load_build_config(name, build_config_path, dictionary_name) -> dict: + # build configs must have a .bzl suffix for Starlark import, so we are forced to do this + # workaround. + _extensions_build_config_spec = spec_from_loader( + name, SourceFileLoader(name, build_config_path)) + + if not isinstance(_extensions_build_config_spec, ModuleSpec): + raise ExtensionsConfigurationError(f"Unable to parse build config {build_config_path}") + extensions_build_config = module_from_spec(_extensions_build_config_spec) + + if not isinstance(_extensions_build_config_spec.loader, Loader): + raise ExtensionsConfigurationError(f"Unable to parse build config {build_config_path}") + + _extensions_build_config_spec.loader.exec_module(extensions_build_config) + return getattr(extensions_build_config, dictionary_name) + def check_fuzzed(self) -> None: if self.robust_to_downstream_count == self.fuzzed_count: return diff --git a/tools/extensions/tests/test_extensions_check.py b/tools/extensions/tests/test_extensions_check.py index 12c7ef9a83814..1a194aa89b612 100644 --- a/tools/extensions/tests/test_extensions_check.py +++ b/tools/extensions/tests/test_extensions_check.py @@ -1,6 +1,6 @@ import types from importlib.machinery import ModuleSpec -from unittest.mock import patch, PropertyMock +from unittest.mock import patch, PropertyMock, call import pytest @@ -26,7 +26,7 @@ def test_extensions_checker_all_extensions(): assert ( result - == set(_configured.keys()) | set(extensions_check.BUILTIN_EXTENSIONS)) + == set(_configured.keys()) | set(extensions_check.BUILTIN_EXTENSIONS) | set(checker.configured_contrib_extensions.keys())) assert "all_extensions" in checker.__dict__ @@ -134,13 +134,13 @@ def test_extensions_metadata(patches, is_dict): checker.metadata assert ( - list(m_utils.from_yaml.call_args) - == [(extensions_check.METADATA_PATH,), {}]) + list(m_utils.from_yaml.call_args_list) + == [call(extensions_check.METADATA_PATH), call(extensions_check.CONTRIB_METADATA_PATH)]) if not is_dict: assert ( e.value.args[0] - == f'Unable to parse configuration: {extensions_check.METADATA_PATH}') + == f'Unable to parse metadata: {extensions_check.METADATA_PATH} {extensions_check.CONTRIB_METADATA_PATH}') return assert "metadata" in checker.__dict__ diff --git a/tools/gen_compilation_database.py b/tools/gen_compilation_database.py index a80fea082417a..8c13694b15d2d 100755 --- a/tools/gen_compilation_database.py +++ b/tools/gen_compilation_database.py @@ -93,6 +93,13 @@ def fix_compilation_database(args, db): parser.add_argument('--include_headers', action='store_true') parser.add_argument('--vscode', action='store_true') parser.add_argument( - 'bazel_targets', nargs='*', default=["//source/...", "//test/...", "//tools/..."]) + 'bazel_targets', + nargs='*', + default=[ + "//source/...", + "//test/...", + "//tools/...", + "//contrib/...", + ]) args = parser.parse_args() fix_compilation_database(args, generate_compilation_database(args)) diff --git a/tools/proto_format/proto_sync.py b/tools/proto_format/proto_sync.py index 52bc5710ce675..9fabe1b9c9eb1 100755 --- a/tools/proto_format/proto_sync.py +++ b/tools/proto_format/proto_sync.py @@ -162,8 +162,13 @@ def get_destination_path(src): if len(matches) != 1: raise RequiresReformatError( "Expect {} has only one package declaration but has {}".format(src, len(matches))) - return pathlib.Path(get_directory_from_package( + dst_path = pathlib.Path(get_directory_from_package( matches[0])).joinpath(src_path.name.split('.')[0] + ".proto") + # contrib API files have the standard namespace but are in a contrib folder for clarity. + # The following prepends contrib for contrib packages so we wind up with the real final path. + if 'contrib' in src: + dst_path = pathlib.Path('contrib').joinpath(dst_path) + return dst_path def get_abs_rel_destination_path(dst_root, src): @@ -395,6 +400,9 @@ def generate_current_api_dir(api_dir, dst_dir): api_dir: the original api directory dst_dir: the api directory to be compared in temporary directory """ + contrib_dst = dst_dir.joinpath("contrib") + shutil.copytree(str(api_dir.joinpath("contrib")), str(contrib_dst)) + dst = dst_dir.joinpath("envoy") shutil.copytree(str(api_dir.joinpath("envoy")), str(dst)) diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index 8ac4ca1c43d60..0636bd08fbc75 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -23,6 +23,7 @@ py_binary( name = "protodoc", srcs = ["protodoc.py"], data = [ + "//contrib:extensions_metadata.yaml", "//docs:protodoc_manifest.yaml", "//docs:v2_mapping.json", "//source/extensions:extensions_metadata.yaml", diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index a95efe31b4b46..a8d45c7ccd1e5 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -56,7 +56,7 @@ .. _extension_{{extension}}: This extension may be referenced by the qualified name ``{{extension}}`` - +{{contrib}} .. note:: {{status}} @@ -77,12 +77,22 @@ .. _extension_category_{{category}}: .. tip:: +{% if extensions %} This extension category has the following known extensions: {% for ext in extensions %} - :ref:`{{ext}} ` {% endfor %} +{% endif %} +{% if contrib_extensions %} + The following extensions are available in :ref:`contrib ` images only: + +{% for ext in contrib_extensions %} + - :ref:`{{ext}} ` +{% endfor %} +{% endif %} + """) # A map from the extension security postures (as defined in the @@ -117,12 +127,20 @@ r = runfiles.Create() EXTENSION_DB = utils.from_yaml(r.Rlocation("envoy/source/extensions/extensions_metadata.yaml")) +CONTRIB_EXTENSION_DB = utils.from_yaml(r.Rlocation("envoy/contrib/extensions_metadata.yaml")) + # create an index of extension categories from extension db -EXTENSION_CATEGORIES = {} -for _k, _v in EXTENSION_DB.items(): - for _cat in _v['categories']: - EXTENSION_CATEGORIES.setdefault(_cat, []).append(_k) +def build_categories(extensions_db): + ret = {} + for _k, _v in extensions_db.items(): + for _cat in _v['categories']: + ret.setdefault(_cat, []).append(_k) + return ret + + +EXTENSION_CATEGORIES = build_categories(EXTENSION_DB) +CONTRIB_EXTENSION_CATEGORIES = build_categories(CONTRIB_EXTENSION_DB) V2_LINK_TEMPLATE = Template( """ @@ -241,18 +259,29 @@ def format_extension(extension): RST formatted extension description. """ try: - extension_metadata = EXTENSION_DB[extension] + extension_metadata = EXTENSION_DB.get(extension, None) + contrib = '' + if extension_metadata is None: + extension_metadata = CONTRIB_EXTENSION_DB[extension] + contrib = """ + +.. note:: + This extension is only available in :ref:`contrib ` images. + +""" status = EXTENSION_STATUS_VALUES.get(extension_metadata.get('status'), '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] categories = extension_metadata["categories"] except KeyError as e: sys.stderr.write( - f"\n\nDid you forget to add '{extension}' to source/extensions/extensions_build_config.bzl " - "or source/extensions/extensions_metadata.yaml?\n\n") + f"\n\nDid you forget to add '{extension}' to extensions_build_config.bzl, " + "extensions_metadata.yaml, contrib_build_config.bzl, " + "or contrib/extensions_metadata.yaml?\n\n") exit(1) # Raising the error buries the above message in tracebacks. return EXTENSION_TEMPLATE.render( extension=extension, + contrib=contrib, status=status, security_posture=security_posture, categories=categories) @@ -267,12 +296,14 @@ def format_extension_category(extension_category): Returns: RST formatted extension category description. """ - try: - extensions = EXTENSION_CATEGORIES[extension_category] - except KeyError as e: + extensions = EXTENSION_CATEGORIES.get(extension_category, []) + contrib_extensions = CONTRIB_EXTENSION_CATEGORIES.get(extension_category, []) + if not extensions and not contrib_extensions: raise ProtodocError(f"\n\nUnable to find extension category: {extension_category}\n\n") return EXTENSION_CATEGORY_TEMPLATE.render( - category=extension_category, extensions=sorted(extensions)) + category=extension_category, + extensions=sorted(extensions), + contrib_extensions=sorted(contrib_extensions)) def format_header_from_file(style, source_code_info, proto_name, v2_link): diff --git a/tools/type_whisperer/proto_build_targets_gen.py b/tools/type_whisperer/proto_build_targets_gen.py index 566febc443f33..2fd33ac0361c3 100644 --- a/tools/type_whisperer/proto_build_targets_gen.py +++ b/tools/type_whisperer/proto_build_targets_gen.py @@ -120,6 +120,11 @@ def is_v3_package(pkg): for desc in type_db.types.values(): pkg = desc.qualified_package if is_v3_package(pkg): + # contrib API files have the standard namespace but are in a contrib folder for clarity. + # The following prepends contrib to the package path which indirectly will produce the + # proper bazel path. + if desc.proto_path.startswith('contrib/'): + pkg = "contrib." + pkg v3_packages.add(pkg) continue if is_v2_package(pkg): From cf1e3213680041e6db551592e2ef74aeea9eae02 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Tue, 10 Aug 2021 17:49:56 +0000 Subject: [PATCH 2/4] comment Signed-off-by: Matt Klein --- CONTRIBUTING.md | 4 +++- EXTENSION_POLICY.md | 2 ++ SECURITY.md | 2 ++ .../arch_overview/security/threat_model.rst | 5 +++++ tools/proto_format/proto_sync.py | 17 +++++++++++++++-- 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7098c2ad46aa9..c59d73c0e7222 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -268,7 +268,9 @@ Other changes will likely include See [EXTENSION_POLICY.md](EXTENSION_POLICY.md) for more information on contrib. Adding a contrib extension mostly mirrors adding a normal extension above. Some differences are noted here: -* API files should be added in `api/contrib/envoy/` +* API files should be added in `api/contrib/envoy/`, but the protos' namespaces should still be as + in normal extensions (which will make file movement easier later if the extension gets promoted + to core). * Build config and metadata should be included in [contrib/contrib_build_config.bzl](contrib/contrib_build_config.bzl) and [contrib/extensions_metadata.yaml](contrib/extensions_metadata.yaml). diff --git a/EXTENSION_POLICY.md b/EXTENSION_POLICY.md index 3fed033ada676..24abecc7055f4 100644 --- a/EXTENSION_POLICY.md +++ b/EXTENSION_POLICY.md @@ -146,3 +146,5 @@ contrib extension is lower than a core extension, with the tradeoff that contrib included by default in the main image builds. Consumers need to pull directly from the contrib images described in the installation guide. Please read the linked document in detail to determine whether contrib extensions are the right choice for a newly proposed extension. + +**NOTE:** Contrib extensions are not eligible for Envoy security team coverage. diff --git a/SECURITY.md b/SECURITY.md index 62a1424a1a983..54fe4981bf431 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -93,6 +93,8 @@ detect issues during their execution on ClusterFuzz. A soak period of 7 days pro guarantee, since we will invoke the security release process for medium or higher severity issues for these older bugs. +**NOTE:** Contrib extensions are not eligible for Envoy security team coverage. + ### Threat model See https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/threat_model. diff --git a/docs/root/intro/arch_overview/security/threat_model.rst b/docs/root/intro/arch_overview/security/threat_model.rst index 66b906f1cb281..93aa6530b545e 100644 --- a/docs/root/intro/arch_overview/security/threat_model.rst +++ b/docs/root/intro/arch_overview/security/threat_model.rst @@ -83,6 +83,11 @@ and do not qualify for treatment under the threat model below. As a consequence, with this model in mind. Security issues related to core code will usually trigger the security release process as described in this document. +.. note:: + + :ref:`contrib ` extensions are noted below and are not officially covered by + the threat model or the Envoy security team. All indications below are best effort. + The following extensions are intended to be hardened against untrusted downstream and upstreams: .. include:: secpos_robust_to_untrusted_downstream_and_upstream.rst diff --git a/tools/proto_format/proto_sync.py b/tools/proto_format/proto_sync.py index 9fabe1b9c9eb1..6eae2561420b1 100755 --- a/tools/proto_format/proto_sync.py +++ b/tools/proto_format/proto_sync.py @@ -41,6 +41,12 @@ 'validate/validate.proto', ] +# Each of the following contrib extensions are allowed to be in the v3 namespace. Indicate why. +CONTRIB_V3_ALLOW_LIST = [ + # Extension moved from core to contrib. + 'envoy.extensions.filters.http.squash.v3', +] + BUILD_FILE_TEMPLATE = string.Template( """# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. @@ -162,11 +168,18 @@ def get_destination_path(src): if len(matches) != 1: raise RequiresReformatError( "Expect {} has only one package declaration but has {}".format(src, len(matches))) - dst_path = pathlib.Path(get_directory_from_package( - matches[0])).joinpath(src_path.name.split('.')[0] + ".proto") + package = matches[0] + dst_path = pathlib.Path( + get_directory_from_package(package)).joinpath(src_path.name.split('.')[0] + ".proto") # contrib API files have the standard namespace but are in a contrib folder for clarity. # The following prepends contrib for contrib packages so we wind up with the real final path. if 'contrib' in src: + if 'v3alpha' not in package and package not in CONTRIB_V3_ALLOW_LIST: + raise ProtoSyncError( + "contrib extension package '{}' does not use v3alpha namespace. " + "Add to CONTRIB_V3_ALLOW_LIST with an explanation if this is on purpose.".format( + package)) + dst_path = pathlib.Path('contrib').joinpath(dst_path) return dst_path From da98409540d37011e26158d8ee64f215e43fbed1 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Wed, 11 Aug 2021 16:42:12 +0000 Subject: [PATCH 3/4] comments Signed-off-by: Matt Klein --- CODEOWNERS | 5 +++- EXTENSION_POLICY.md | 4 ++- bazel/README.md | 7 ++++++ contrib/contrib_build_config.bzl | 2 +- .../filters/http}/source/BUILD | 4 +-- .../filters/http}/source/config.cc | 8 +++--- .../filters/http}/source/config.h | 4 +-- .../filters/http}/source/squash_filter.cc | 6 ++--- .../filters/http}/source/squash_filter.h | 2 +- .../squash => squash/filters/http}/test/BUILD | 10 ++++---- .../filters/http}/test/config_test.cc | 6 ++--- .../test/squash_filter_integration_test.cc | 0 .../filters/http}/test/squash_filter_test.cc | 4 +-- tools/code_format/check_format.py | 25 ++++++++++++++++--- 14 files changed, 58 insertions(+), 29 deletions(-) rename contrib/{filters/http/squash => squash/filters/http}/source/BUILD (88%) rename contrib/{filters/http/squash => squash/filters/http}/source/config.cc (82%) rename contrib/{filters/http/squash => squash/filters/http}/source/config.h (86%) rename contrib/{filters/http/squash => squash/filters/http}/source/squash_filter.cc (98%) rename contrib/{filters/http/squash => squash/filters/http}/source/squash_filter.h (99%) rename contrib/{filters/http/squash => squash/filters/http}/test/BUILD (75%) rename contrib/{filters/http/squash => squash/filters/http}/test/config_test.cc (89%) rename contrib/{filters/http/squash => squash/filters/http}/test/squash_filter_integration_test.cc (100%) rename contrib/{filters/http/squash => squash/filters/http}/test/squash_filter_test.cc (99%) diff --git a/CODEOWNERS b/CODEOWNERS index 2f689fab7a165..47d9ddfcee11e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -119,7 +119,6 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/http/router @alyssawilk @mattklein123 @snowp /*/extensions/filters/http/grpc_web @fengli79 @lizan /*/extensions/filters/http/grpc_stats @kyessenov @lizan -/*/extensions/filters/http/squash @yuval-k @alyssawilk /*/extensions/filters/common/original_src @klarose @snowp /*/extensions/filters/listener/tls_inspector @ggreenway @asraa /*/extensions/grpc_credentials/example @wozz @htuch @@ -201,3 +200,7 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp /*/extensions/matching/input_matchers/ip @aguinet @snowp # Kafka /*/extensions/filters/network/kafka @mattklein123 @adamkotwasinski + +# Contrib +/contrib/exe/ @mattklein123 @lizan +/contrib/squash/ @yuval-k @alyssawilk diff --git a/EXTENSION_POLICY.md b/EXTENSION_POLICY.md index 24abecc7055f4..7ef47bcd6cf13 100644 --- a/EXTENSION_POLICY.md +++ b/EXTENSION_POLICY.md @@ -141,10 +141,12 @@ case we need to install an extension point, which can be done as follows: ## Contrib extensions As described in [this document](https://docs.google.com/document/d/1yl7GOZK1TDm_7vxQvt8UQEAu07UQFru1uEKXM6ZZg_g/edit#), -Envoy allows an alternate path to adding extensions called "contrib." The barrier to entry for a +Envoy allows an alternate path to adding extensions called `contrib/`. The barrier to entry for a contrib extension is lower than a core extension, with the tradeoff that contrib extensions are not included by default in the main image builds. Consumers need to pull directly from the contrib images described in the installation guide. Please read the linked document in detail to determine whether contrib extensions are the right choice for a newly proposed extension. **NOTE:** Contrib extensions are not eligible for Envoy security team coverage. +**NOTE:** As per the linked Google Doc, contrib extensions generally should use `v3alpha` to avoid +requiring API shepherd reviews. diff --git a/bazel/README.md b/bazel/README.md index b8328b770269b..cf575fa58c197 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -684,6 +684,13 @@ If you're building from a custom build repository, the parameters need to prefix You may persist those options in `user.bazelrc` in Envoy repo or your `.bazelrc`. +Contrib extensions can be enabled and disabled similarly to above when building the contrib +executable. For example: + +`bazel build //contrib/exe:envoy-static --//contrib/squash/filters/http/source:enabled=false` + +Will disable the squash extension when building the contrib executable. + ## Customize extension build config You can also use the following procedure to customize the extensions for your build: diff --git a/contrib/contrib_build_config.bzl b/contrib/contrib_build_config.bzl index ddf21dcb07329..9c51607746a12 100644 --- a/contrib/contrib_build_config.bzl +++ b/contrib/contrib_build_config.bzl @@ -4,5 +4,5 @@ CONTRIB_EXTENSIONS = { # HTTP filters # - "envoy.filters.http.squash": "//contrib/filters/http/squash/source:config", + "envoy.filters.http.squash": "//contrib/squash/filters/http/source:config", } diff --git a/contrib/filters/http/squash/source/BUILD b/contrib/squash/filters/http/source/BUILD similarity index 88% rename from contrib/filters/http/squash/source/BUILD rename to contrib/squash/filters/http/source/BUILD index 897149d5137fc..08be4d1b989a3 100644 --- a/contrib/filters/http/squash/source/BUILD +++ b/contrib/squash/filters/http/source/BUILD @@ -29,7 +29,7 @@ envoy_cc_library( "//source/common/http:utility_lib", "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", - "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/squash/filters/http/v3:pkg_cc_proto", ], ) @@ -42,6 +42,6 @@ envoy_cc_contrib_extension( "//envoy/registry", "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", - "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/squash/filters/http/v3:pkg_cc_proto", ], ) diff --git a/contrib/filters/http/squash/source/config.cc b/contrib/squash/filters/http/source/config.cc similarity index 82% rename from contrib/filters/http/squash/source/config.cc rename to contrib/squash/filters/http/source/config.cc index 7d65a5416ed66..62a512b17c0e1 100644 --- a/contrib/filters/http/squash/source/config.cc +++ b/contrib/squash/filters/http/source/config.cc @@ -1,13 +1,13 @@ -#include "contrib/filters/http/squash/source/config.h" +#include "contrib/squash/filters/http/source/config.h" #include "envoy/registry/registry.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" -#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" -#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" -#include "contrib/filters/http/squash/source/squash_filter.h" +#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" +#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.validate.h" +#include "contrib/squash/filters/http/source/squash_filter.h" namespace Envoy { namespace Extensions { diff --git a/contrib/filters/http/squash/source/config.h b/contrib/squash/filters/http/source/config.h similarity index 86% rename from contrib/filters/http/squash/source/config.h rename to contrib/squash/filters/http/source/config.h index feff75eaf8b0e..8fa753acd6879 100644 --- a/contrib/filters/http/squash/source/config.h +++ b/contrib/squash/filters/http/source/config.h @@ -2,8 +2,8 @@ #include "source/extensions/filters/http/common/factory_base.h" -#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" -#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" +#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" +#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.validate.h" namespace Envoy { namespace Extensions { diff --git a/contrib/filters/http/squash/source/squash_filter.cc b/contrib/squash/filters/http/source/squash_filter.cc similarity index 98% rename from contrib/filters/http/squash/source/squash_filter.cc rename to contrib/squash/filters/http/source/squash_filter.cc index a906505bf0aca..2d78a44e918eb 100644 --- a/contrib/filters/http/squash/source/squash_filter.cc +++ b/contrib/squash/filters/http/source/squash_filter.cc @@ -1,4 +1,4 @@ -#include "contrib/filters/http/squash/source/squash_filter.h" +#include "contrib/squash/filters/http/source/squash_filter.h" #include @@ -15,7 +15,7 @@ #include "source/common/protobuf/utility.h" #include "absl/container/fixed_array.h" -#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" namespace Envoy { namespace Extensions { @@ -91,7 +91,7 @@ void SquashFilterConfig::updateTemplateInValue(ProtobufWkt::Value& curvalue) { To interpolate an environment variable named ENV, add '{{ ENV }}' (without the quotes, with the spaces) to the template string. - See api/envoy/extensions/filters/http/squash/v3/squash.proto for the motivation on why this is + See api/envoy/extensions/squash/filters/http/v3/squash.proto for the motivation on why this is needed. */ std::string SquashFilterConfig::replaceEnv(const std::string& attachment_template) { diff --git a/contrib/filters/http/squash/source/squash_filter.h b/contrib/squash/filters/http/source/squash_filter.h similarity index 99% rename from contrib/filters/http/squash/source/squash_filter.h rename to contrib/squash/filters/http/source/squash_filter.h index df1688210ac4b..86ba746e531d0 100644 --- a/contrib/filters/http/squash/source/squash_filter.h +++ b/contrib/squash/filters/http/source/squash_filter.h @@ -11,7 +11,7 @@ #include "source/common/protobuf/protobuf.h" #include "absl/types/optional.h" -#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" namespace Envoy { namespace Extensions { diff --git a/contrib/filters/http/squash/test/BUILD b/contrib/squash/filters/http/test/BUILD similarity index 75% rename from contrib/filters/http/squash/test/BUILD rename to contrib/squash/filters/http/test/BUILD index 862ff5a09e720..b8c4e9ae8ce81 100644 --- a/contrib/filters/http/squash/test/BUILD +++ b/contrib/squash/filters/http/test/BUILD @@ -12,7 +12,7 @@ envoy_cc_test( name = "squash_filter_test", srcs = ["squash_filter_test.cc"], deps = [ - "//contrib/filters/http/squash/source:squash_filter_lib", + "//contrib/squash/filters/http/source:squash_filter_lib", "//envoy/event:dispatcher_interface", "//source/common/http:header_map_lib", "//source/common/stats:stats_lib", @@ -20,7 +20,7 @@ envoy_cc_test( "//test/mocks/server:factory_context_mocks", "//test/mocks/upstream:cluster_manager_mocks", "//test/test_common:environment_lib", - "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/squash/filters/http/v3:pkg_cc_proto", ], ) @@ -28,7 +28,7 @@ envoy_cc_test( name = "squash_filter_integration_test", srcs = ["squash_filter_integration_test.cc"], deps = [ - "//contrib/filters/http/squash/source:config", + "//contrib/squash/filters/http/source:config", "//test/integration:http_integration_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", ], @@ -38,9 +38,9 @@ envoy_cc_test( name = "config_test", srcs = ["config_test.cc"], deps = [ - "//contrib/filters/http/squash/source:config", + "//contrib/squash/filters/http/source:config", "//test/mocks/server:factory_context_mocks", "//test/test_common:utility_lib", - "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/squash/filters/http/v3:pkg_cc_proto", ], ) diff --git a/contrib/filters/http/squash/test/config_test.cc b/contrib/squash/filters/http/test/config_test.cc similarity index 89% rename from contrib/filters/http/squash/test/config_test.cc rename to contrib/squash/filters/http/test/config_test.cc index 7561664845a26..163c57324e7cf 100644 --- a/contrib/filters/http/squash/test/config_test.cc +++ b/contrib/squash/filters/http/test/config_test.cc @@ -1,9 +1,9 @@ #include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" -#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" -#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" -#include "contrib/filters/http/squash/source/config.h" +#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" +#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.validate.h" +#include "contrib/squash/filters/http/source/config.h" #include "gmock/gmock.h" #include "gtest/gtest.h" diff --git a/contrib/filters/http/squash/test/squash_filter_integration_test.cc b/contrib/squash/filters/http/test/squash_filter_integration_test.cc similarity index 100% rename from contrib/filters/http/squash/test/squash_filter_integration_test.cc rename to contrib/squash/filters/http/test/squash_filter_integration_test.cc diff --git a/contrib/filters/http/squash/test/squash_filter_test.cc b/contrib/squash/filters/http/test/squash_filter_test.cc similarity index 99% rename from contrib/filters/http/squash/test/squash_filter_test.cc rename to contrib/squash/filters/http/test/squash_filter_test.cc index 82ed3181e07c0..0e6247f5b28d3 100644 --- a/contrib/filters/http/squash/test/squash_filter_test.cc +++ b/contrib/squash/filters/http/test/squash_filter_test.cc @@ -10,8 +10,8 @@ #include "test/test_common/environment.h" #include "test/test_common/utility.h" -#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" -#include "contrib/filters/http/squash/source/squash_filter.h" +#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" +#include "contrib/squash/filters/http/source/squash_filter.h" #include "fmt/format.h" #include "gmock/gmock.h" #include "gtest/gtest.h" diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index 078715a96091c..2b65d6b904db6 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -98,8 +98,8 @@ "./source/common/common/regex.cc", "./source/common/stats/tag_extractor_impl.h", "./source/common/stats/tag_extractor_impl.cc", "./source/common/formatter/substitution_formatter.cc", - "./contrib/filters/http/squash/source/squash_filter.h", - "./contrib/filters/http/squash/source/squash_filter.cc", "./source/server/admin/utils.h", + "./contrib/squash/filters/http/source/squash_filter.h", + "./contrib/squash/filters/http/source/squash_filter.cc", "./source/server/admin/utils.h", "./source/server/admin/utils.cc", "./source/server/admin/stats_handler.h", "./source/server/admin/stats_handler.cc", "./source/server/admin/prometheus_stats.h", "./source/server/admin/prometheus_stats.cc", "./tools/clang_tools/api_booster/main.cc", @@ -177,6 +177,7 @@ HISTOGRAM_SI_SUFFIX_REGEX = re.compile(r"(?<=HISTOGRAM\()[a-zA-Z0-9_]+_(b|kb|mb|ns|us|ms|s)(?=,)") TEST_NAME_STARTING_LOWER_CASE_REGEX = re.compile(r"TEST(_.\(.*,\s|\()[a-z].*\)\s\{") EXTENSIONS_CODEOWNERS_REGEX = re.compile(r'.*(extensions[^@]*\s+)(@.*)') +CONTRIB_CODEOWNERS_REGEX = re.compile(r'(/contrib/\w+/\s+)(@.*)') COMMENT_REGEX = re.compile(r"//|\*") DURATION_VALUE_REGEX = re.compile(r'\b[Dd]uration\(([0-9.]+)') PROTO_VALIDATION_STRING = re.compile(r'\bmin_bytes\b') @@ -1087,13 +1088,20 @@ def check_format_visitor(self, arg, dir_name, names): # Sanity check CODEOWNERS. This doesn't need to be done in a multi-threaded # manner as it is a small and limited list. source_prefix = './source/' - full_prefix = './source/extensions/' + core_extensions_full_prefix = './source/extensions/' # Check to see if this directory is a subdir under /source/extensions # Also ignore top level directories under /source/extensions since we don't # need owners for source/extensions/access_loggers etc, just the subdirectories. - if dir_name.startswith(full_prefix) and '/' in dir_name[len(full_prefix):]: + if dir_name.startswith( + core_extensions_full_prefix) and '/' in dir_name[len(core_extensions_full_prefix):]: self.check_owners(dir_name[len(source_prefix):], owned_directories, error_messages) + # For contrib extensions we track ownership at the top level only. + contrib_prefix = './contrib/' + if dir_name.startswith(contrib_prefix): + top_level = pathlib.PurePath('/', *pathlib.PurePath(dir_name).parts[:2], '/') + self.check_owners(str(top_level), owned_directories, error_messages) + for file_name in names: if dir_name.startswith("./api") and self.is_starlark_file(file_name): result = pool.apply_async( @@ -1215,6 +1223,15 @@ def owned_directories(error_messages): "Extensions require at least one maintainer OWNER:\n" " {}".format(line)) + m = CONTRIB_CODEOWNERS_REGEX.search(line) + if m is not None and not line.startswith('#'): + owned.append(m.group(1).strip()) + owners = re.findall('@\S+', m.group(2).strip()) + if len(owners) < 2: + error_messages.append( + "Contrib extensions require at least 2 owners in CODEOWNERS:\n" + " {}".format(line)) + return owned except IOError: return [] # for the check format tests. From 3714199d43035ecc72fb1dc942f10903f95c65de Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Wed, 11 Aug 2021 19:09:48 +0000 Subject: [PATCH 4/4] fix Signed-off-by: Matt Klein --- contrib/squash/filters/http/source/BUILD | 4 ++-- contrib/squash/filters/http/source/config.cc | 4 ++-- contrib/squash/filters/http/source/config.h | 4 ++-- contrib/squash/filters/http/source/squash_filter.cc | 2 +- contrib/squash/filters/http/source/squash_filter.h | 2 +- contrib/squash/filters/http/test/BUILD | 4 ++-- contrib/squash/filters/http/test/config_test.cc | 4 ++-- contrib/squash/filters/http/test/squash_filter_test.cc | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/contrib/squash/filters/http/source/BUILD b/contrib/squash/filters/http/source/BUILD index 08be4d1b989a3..897149d5137fc 100644 --- a/contrib/squash/filters/http/source/BUILD +++ b/contrib/squash/filters/http/source/BUILD @@ -29,7 +29,7 @@ envoy_cc_library( "//source/common/http:utility_lib", "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", - "@envoy_api//contrib/envoy/extensions/squash/filters/http/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) @@ -42,6 +42,6 @@ envoy_cc_contrib_extension( "//envoy/registry", "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", - "@envoy_api//contrib/envoy/extensions/squash/filters/http/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) diff --git a/contrib/squash/filters/http/source/config.cc b/contrib/squash/filters/http/source/config.cc index 62a512b17c0e1..cec853979f2e6 100644 --- a/contrib/squash/filters/http/source/config.cc +++ b/contrib/squash/filters/http/source/config.cc @@ -5,8 +5,8 @@ #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" -#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" -#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.validate.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" #include "contrib/squash/filters/http/source/squash_filter.h" namespace Envoy { diff --git a/contrib/squash/filters/http/source/config.h b/contrib/squash/filters/http/source/config.h index 8fa753acd6879..feff75eaf8b0e 100644 --- a/contrib/squash/filters/http/source/config.h +++ b/contrib/squash/filters/http/source/config.h @@ -2,8 +2,8 @@ #include "source/extensions/filters/http/common/factory_base.h" -#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" -#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.validate.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" namespace Envoy { namespace Extensions { diff --git a/contrib/squash/filters/http/source/squash_filter.cc b/contrib/squash/filters/http/source/squash_filter.cc index 2d78a44e918eb..41fc37e5f7613 100644 --- a/contrib/squash/filters/http/source/squash_filter.cc +++ b/contrib/squash/filters/http/source/squash_filter.cc @@ -15,7 +15,7 @@ #include "source/common/protobuf/utility.h" #include "absl/container/fixed_array.h" -#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" namespace Envoy { namespace Extensions { diff --git a/contrib/squash/filters/http/source/squash_filter.h b/contrib/squash/filters/http/source/squash_filter.h index 86ba746e531d0..df1688210ac4b 100644 --- a/contrib/squash/filters/http/source/squash_filter.h +++ b/contrib/squash/filters/http/source/squash_filter.h @@ -11,7 +11,7 @@ #include "source/common/protobuf/protobuf.h" #include "absl/types/optional.h" -#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" namespace Envoy { namespace Extensions { diff --git a/contrib/squash/filters/http/test/BUILD b/contrib/squash/filters/http/test/BUILD index b8c4e9ae8ce81..a2a9b49f4c9ad 100644 --- a/contrib/squash/filters/http/test/BUILD +++ b/contrib/squash/filters/http/test/BUILD @@ -20,7 +20,7 @@ envoy_cc_test( "//test/mocks/server:factory_context_mocks", "//test/mocks/upstream:cluster_manager_mocks", "//test/test_common:environment_lib", - "@envoy_api//contrib/envoy/extensions/squash/filters/http/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) @@ -41,6 +41,6 @@ envoy_cc_test( "//contrib/squash/filters/http/source:config", "//test/mocks/server:factory_context_mocks", "//test/test_common:utility_lib", - "@envoy_api//contrib/envoy/extensions/squash/filters/http/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) diff --git a/contrib/squash/filters/http/test/config_test.cc b/contrib/squash/filters/http/test/config_test.cc index 163c57324e7cf..613dc5b67e9ad 100644 --- a/contrib/squash/filters/http/test/config_test.cc +++ b/contrib/squash/filters/http/test/config_test.cc @@ -1,8 +1,8 @@ #include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" -#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" -#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.validate.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" #include "contrib/squash/filters/http/source/config.h" #include "gmock/gmock.h" #include "gtest/gtest.h" diff --git a/contrib/squash/filters/http/test/squash_filter_test.cc b/contrib/squash/filters/http/test/squash_filter_test.cc index 0e6247f5b28d3..756e83e402fb5 100644 --- a/contrib/squash/filters/http/test/squash_filter_test.cc +++ b/contrib/squash/filters/http/test/squash_filter_test.cc @@ -10,7 +10,7 @@ #include "test/test_common/environment.h" #include "test/test_common/utility.h" -#include "contrib/envoy/extensions/squash/filters/http/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" #include "contrib/squash/filters/http/source/squash_filter.h" #include "fmt/format.h" #include "gmock/gmock.h"