diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index 527305ef41d83..e5f2904cfb54f 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -310,6 +310,13 @@ stages: itemPattern: "bazel.release/envoy_binary.tar.gz" downloadType: single targetPath: $(Build.StagingDirectory) + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release" + itemPattern: "bazel.release/envoy-contrib_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) - task: DownloadBuildArtifacts@0 inputs: buildType: current @@ -317,10 +324,19 @@ stages: itemPattern: "bazel.release.arm64/envoy_binary.tar.gz" downloadType: single targetPath: $(Build.StagingDirectory) + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release.arm64" + itemPattern: "bazel.release.arm64/envoy-contrib_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) - bash: | set -e mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64 + tar zxf $(Build.StagingDirectory)/bazel.release/envoy-contrib_binary.tar.gz -C ./linux/amd64 mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64 + tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy-contrib_binary.tar.gz -C ./linux/arm64 ci/docker_ci.sh workingDirectory: $(Build.SourcesDirectory) env: diff --git a/.dockerignore b/.dockerignore index ee59c60405f33..e0ef1fd120316 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,8 +3,8 @@ !/ci !/configs/google-vrp !/configs/*yaml -!/linux/amd64/build_release* -!/linux/arm64/build_release* +!/linux/amd64/build_*release* +!/linux/arm64/build_*release* !/local !/test/config/integration/certs !/windows diff --git a/BUILD b/BUILD index 7ef982f5621bc..9e35562c085fb 100644 --- a/BUILD +++ b/BUILD @@ -33,3 +33,10 @@ package_group( "//test/extensions/...", ], ) + +package_group( + name = "contrib_library", + packages = [ + "//contrib/...", + ], +) diff --git a/CODEOWNERS b/CODEOWNERS index 2f689fab7a165..47d9ddfcee11e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -119,7 +119,6 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/http/router @alyssawilk @mattklein123 @snowp /*/extensions/filters/http/grpc_web @fengli79 @lizan /*/extensions/filters/http/grpc_stats @kyessenov @lizan -/*/extensions/filters/http/squash @yuval-k @alyssawilk /*/extensions/filters/common/original_src @klarose @snowp /*/extensions/filters/listener/tls_inspector @ggreenway @asraa /*/extensions/grpc_credentials/example @wozz @htuch @@ -201,3 +200,7 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp /*/extensions/matching/input_matchers/ip @aguinet @snowp # Kafka /*/extensions/filters/network/kafka @mattklein123 @adamkotwasinski + +# Contrib +/contrib/exe/ @mattklein123 @lizan +/contrib/squash/ @yuval-k @alyssawilk diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38cded6eaf8b5..c59d73c0e7222 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -262,7 +262,17 @@ Other changes will likely include * Editing [source/extensions/extensions_metadata.yaml](source/extensions/extensions_metadata.yaml) to include metadata for the new extensions * Editing [docs/root/api-v3/config/config.rst](docs/root/api-v3/config/config.rst) to add area/area * Adding `docs/root/api-v3/config/area/area.rst` to add a table of contents for the API docs - * Adding `source/extensions/area/well_known_names.h` for registered plugins + +# Adding contrib extensions + +See [EXTENSION_POLICY.md](EXTENSION_POLICY.md) for more information on contrib. Adding a contrib +extension mostly mirrors adding a normal extension above. Some differences are noted here: + +* API files should be added in `api/contrib/envoy/`, but the protos' namespaces should still be as + in normal extensions (which will make file movement easier later if the extension gets promoted + to core). +* Build config and metadata should be included in [contrib/contrib_build_config.bzl](contrib/contrib_build_config.bzl) + and [contrib/extensions_metadata.yaml](contrib/extensions_metadata.yaml). # DCO: Sign your work diff --git a/EXTENSION_POLICY.md b/EXTENSION_POLICY.md index 39d41f14f875f..7ef47bcd6cf13 100644 --- a/EXTENSION_POLICY.md +++ b/EXTENSION_POLICY.md @@ -137,3 +137,16 @@ case we need to install an extension point, which can be done as follows: 3. Update [extending envoy](docs/root/extending/extending.rst) to list the new extension point and add any documentation explaining the extension point. At the very least this should link to the corresponding proto. + +## Contrib extensions + +As described in [this document](https://docs.google.com/document/d/1yl7GOZK1TDm_7vxQvt8UQEAu07UQFru1uEKXM6ZZg_g/edit#), +Envoy allows an alternate path to adding extensions called `contrib/`. The barrier to entry for a +contrib extension is lower than a core extension, with the tradeoff that contrib extensions are not +included by default in the main image builds. Consumers need to pull directly from the contrib +images described in the installation guide. Please read the linked document in detail to determine +whether contrib extensions are the right choice for a newly proposed extension. + +**NOTE:** Contrib extensions are not eligible for Envoy security team coverage. +**NOTE:** As per the linked Google Doc, contrib extensions generally should use `v3alpha` to avoid +requiring API shepherd reviews. diff --git a/REPO_LAYOUT.md b/REPO_LAYOUT.md index c29c0b518c3f7..070c3d6683753 100644 --- a/REPO_LAYOUT.md +++ b/REPO_LAYOUT.md @@ -66,7 +66,6 @@ Not every directory within test is described below, but a few highlights: We maintain a very specific code and namespace layout for extensions. This aids in discovering code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNERS). - * All extensions are either registered in [all_extensions.bzl](source/extensions/all_extensions.bzl) or [extensions_build_config.bzl](source/extensions/extensions_build_config.bzl). The former is for extensions that cannot be removed from the primary Envoy build. The latter is for extensions @@ -124,3 +123,14 @@ code/extensions, and allows us specify extension owners in [CODEOWNERS](CODEOWNE code that is used by both HTTP and network filters. Common code used only by two HTTP filters would be found in `filters/http/common/`. Common code should be placed in a common namespace. E.g., `Envoy::Extensions::Filters::Common`. + +## [contrib](contrib/) layout + +This directory contains contrib extensions. See [EXTENSION_POLICY.md](EXTENSION_POLICY.md) for +more information. + +* [contrib/exe/](contrib/exe/): The default executable for contrib. This is similar to the + `envoy-static` target but also includes all contrib extensions, and is used to produce the + contrib image targets. +* [contrib/...](contrib/): The rest of this directory mirrors the [source/extensions](source/extensions/) + layout. Contrib extensions are placed here. diff --git a/SECURITY.md b/SECURITY.md index 62a1424a1a983..54fe4981bf431 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -93,6 +93,8 @@ detect issues during their execution on ClusterFuzz. A soak period of 7 days pro guarantee, since we will invoke the security release process for medium or higher severity issues for these older bugs. +**NOTE:** Contrib extensions are not eligible for Envoy security team coverage. + ### Threat model See https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/security/threat_model. diff --git a/api/BUILD b/api/BUILD index 24e0ea0de018a..044bc7d137a88 100644 --- a/api/BUILD +++ b/api/BUILD @@ -57,6 +57,7 @@ proto_library( name = "v3_protos", visibility = ["//visibility:public"], deps = [ + "//contrib/envoy/extensions/filters/http/squash/v3:pkg", "//envoy/admin/v3:pkg", "//envoy/config/accesslog/v3:pkg", "//envoy/config/bootstrap/v3:pkg", @@ -143,7 +144,6 @@ proto_library( "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", "//envoy/extensions/filters/http/set_metadata/v3:pkg", - "//envoy/extensions/filters/http/squash/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", "//envoy/extensions/filters/http/wasm/v3:pkg", "//envoy/extensions/filters/listener/http_inspector/v3:pkg", diff --git a/api/envoy/extensions/filters/http/squash/v3/BUILD b/api/contrib/envoy/extensions/filters/http/squash/v3/BUILD similarity index 100% rename from api/envoy/extensions/filters/http/squash/v3/BUILD rename to api/contrib/envoy/extensions/filters/http/squash/v3/BUILD diff --git a/api/envoy/extensions/filters/http/squash/v3/squash.proto b/api/contrib/envoy/extensions/filters/http/squash/v3/squash.proto similarity index 100% rename from api/envoy/extensions/filters/http/squash/v3/squash.proto rename to api/contrib/envoy/extensions/filters/http/squash/v3/squash.proto diff --git a/api/test/validate/BUILD b/api/test/validate/BUILD index 049780c2ec036..79b6d6d91da23 100644 --- a/api/test/validate/BUILD +++ b/api/test/validate/BUILD @@ -20,7 +20,6 @@ api_cc_test( "@envoy_api//envoy/extensions/filters/http/ip_tagging/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/lua/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/router/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/mongo_proxy/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/redis_proxy/v3:pkg_cc_proto", diff --git a/api/test/validate/pgv_test.cc b/api/test/validate/pgv_test.cc index feee839c6bfaf..75f7692ba3e69 100644 --- a/api/test/validate/pgv_test.cc +++ b/api/test/validate/pgv_test.cc @@ -19,7 +19,6 @@ #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.validate.h" #include "envoy/extensions/filters/http/lua/v3/lua.pb.validate.h" #include "envoy/extensions/filters/http/router/v3/router.pb.validate.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.validate.h" #include "envoy/extensions/filters/network/mongo_proxy/v3/mongo_proxy.pb.validate.h" #include "envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.pb.validate.h" diff --git a/api/versioning/BUILD b/api/versioning/BUILD index ef972635a9585..33e48b6488fae 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -9,6 +9,7 @@ proto_library( name = "active_protos", visibility = ["//visibility:public"], deps = [ + "//contrib/envoy/extensions/filters/http/squash/v3:pkg", "//envoy/admin/v3:pkg", "//envoy/config/accesslog/v3:pkg", "//envoy/config/bootstrap/v3:pkg", @@ -95,7 +96,6 @@ proto_library( "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", "//envoy/extensions/filters/http/set_metadata/v3:pkg", - "//envoy/extensions/filters/http/squash/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", "//envoy/extensions/filters/http/wasm/v3:pkg", "//envoy/extensions/filters/listener/http_inspector/v3:pkg", diff --git a/bazel/README.md b/bazel/README.md index b8c2b0a03f8df..cf575fa58c197 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -668,7 +668,10 @@ The following optional features can be enabled on the Bazel build command-line: Envoy uses a modular build which allows extensions to be removed if they are not needed or desired. Extensions that can be removed are contained in -[extensions_build_config.bzl](../source/extensions/extensions_build_config.bzl). +[extensions_build_config.bzl](../source/extensions/extensions_build_config.bzl). Contrib build +extensions are contained in [contrib_build_config.bzl](../contrib/contrib_build_config.bzl). Note +that contrib extensions are only included by default when building the contrib executable and in +the default contrib images pushed to Docker Hub. The extensions disabled by default can be enabled by adding the following parameter to Bazel, for example to enable `envoy.filters.http.kill_request` extension, add `--//source/extensions/filters/http/kill_request:enabled`. @@ -681,6 +684,13 @@ If you're building from a custom build repository, the parameters need to prefix You may persist those options in `user.bazelrc` in Envoy repo or your `.bazelrc`. +Contrib extensions can be enabled and disabled similarly to above when building the contrib +executable. For example: + +`bazel build //contrib/exe:envoy-static --//contrib/squash/filters/http/source:enabled=false` + +Will disable the squash extension when building the contrib executable. + ## Customize extension build config You can also use the following procedure to customize the extensions for your build: @@ -719,6 +729,11 @@ local_repository( ... ``` +When performing custom builds, it is acceptable to include contrib extensions as well. This can +be done by including the desired Bazel paths from [contrib_build_config.bzl](../contrib/contrib_build_config.bzl) +into the overriden `extensions_build_config.bzl`. (There is no need to specifically perform +a contrib build to include a contrib extension.) + ## Extra extensions If you are building your own Envoy extensions or custom Envoy builds and encounter visibility diff --git a/bazel/api_binding.bzl b/bazel/api_binding.bzl index 550685f6299a4..56d7c95f08a8d 100644 --- a/bazel/api_binding.bzl +++ b/bazel/api_binding.bzl @@ -8,6 +8,7 @@ def _default_envoy_api_impl(ctx): "test", "tools", "versioning", + "contrib", ] for d in api_dirs: ctx.symlink(ctx.path(ctx.attr.envoy_root).dirname.get_child(ctx.attr.reldir).get_child(d), d) diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index a0463ab770d04..4d671ab9562fa 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -6,6 +6,7 @@ load(":envoy_internal.bzl", "envoy_external_dep_path") load( ":envoy_library.bzl", _envoy_basic_cc_library = "envoy_basic_cc_library", + _envoy_cc_contrib_extension = "envoy_cc_contrib_extension", _envoy_cc_extension = "envoy_cc_extension", _envoy_cc_library = "envoy_cc_library", _envoy_cc_linux_library = "envoy_cc_linux_library", @@ -50,8 +51,8 @@ load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") def envoy_package(): native.package(default_visibility = ["//visibility:public"]) -def envoy_extension_package(enabled_default = True): - native.package(default_visibility = EXTENSION_PACKAGE_VISIBILITY) +def envoy_extension_package(enabled_default = True, default_visibility = EXTENSION_PACKAGE_VISIBILITY): + native.package(default_visibility = default_visibility) bool_flag( name = "enabled", @@ -63,6 +64,9 @@ def envoy_extension_package(enabled_default = True): flag_values = {":enabled": "True"}, ) +def envoy_contrib_package(): + envoy_extension_package(default_visibility = ["//:contrib_library"]) + # A genrule variant that can output a directory. This is useful when doing things like # generating a fuzz corpus mechanically. def _envoy_directory_genrule_impl(ctx): @@ -220,6 +224,7 @@ envoy_cc_binary = _envoy_cc_binary # Library wrappers (from envoy_library.bzl) envoy_basic_cc_library = _envoy_basic_cc_library envoy_cc_extension = _envoy_cc_extension +envoy_cc_contrib_extension = _envoy_cc_contrib_extension envoy_cc_library = _envoy_cc_library envoy_cc_linux_library = _envoy_cc_linux_library envoy_cc_posix_library = _envoy_cc_posix_library diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index c29f35b160c0c..ac74d1be29c96 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -71,6 +71,14 @@ def envoy_cc_extension( visibility = visibility, ) +def envoy_cc_contrib_extension( + name, + tags = [], + extra_visibility = [], + visibility = ["//:contrib_library"], + **kwargs): + envoy_cc_extension(name, tags, extra_visibility, visibility, **kwargs) + # Envoy C++ library targets should be specified with this function. def envoy_cc_library( name, diff --git a/ci/Dockerfile-envoy b/ci/Dockerfile-envoy index 1c7711d540668..ac3cd3e74c12c 100644 --- a/ci/Dockerfile-envoy +++ b/ci/Dockerfile-envoy @@ -13,11 +13,12 @@ RUN apt-get update && apt-get upgrade -y \ RUN mkdir -p /etc/envoy +ARG ENVOY_BINARY=envoy ARG ENVOY_BINARY_SUFFIX=_stripped -ADD ${TARGETPLATFORM}/build_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ +ADD ${TARGETPLATFORM}/build_${ENVOY_BINARY}_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ ADD configs/envoyproxy_io_proxy.yaml /etc/envoy/envoy.yaml -ADD ${TARGETPLATFORM}/build_release/su-exec /usr/local/bin/ +ADD ${TARGETPLATFORM}/build_${ENVOY_BINARY}_release/su-exec /usr/local/bin/ RUN chown root:root /usr/local/bin/su-exec && adduser --group --system envoy EXPOSE 10000 diff --git a/ci/Dockerfile-envoy-alpine b/ci/Dockerfile-envoy-alpine index 18b5aba78f4f2..36a23f6e3a2aa 100644 --- a/ci/Dockerfile-envoy-alpine +++ b/ci/Dockerfile-envoy-alpine @@ -6,7 +6,7 @@ RUN apk add --no-cache shadow su-exec \ && addgroup -S envoy && adduser --no-create-home -S envoy -G envoy ARG ENVOY_BINARY_SUFFIX=_stripped -ADD linux/amd64/build_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ +ADD linux/amd64/build_envoy_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ EXPOSE 10000 diff --git a/ci/Dockerfile-envoy-distroless b/ci/Dockerfile-envoy-distroless index 630ccac08304e..d2647f7b38557 100644 --- a/ci/Dockerfile-envoy-distroless +++ b/ci/Dockerfile-envoy-distroless @@ -3,7 +3,7 @@ FROM gcr.io/distroless/base-debian10:nonroot ADD configs/envoyproxy_io_proxy.yaml /etc/envoy/envoy.yaml ARG ENVOY_BINARY_SUFFIX=_stripped -ADD linux/amd64/build_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ +ADD linux/amd64/build_envoy_release${ENVOY_BINARY_SUFFIX}/* /usr/local/bin/ EXPOSE 10000 diff --git a/ci/build_setup.sh b/ci/build_setup.sh index cedbda30a9e86..977c6477d7dc2 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -10,6 +10,8 @@ export PPROF_PATH=/thirdparty_build/bin/pprof [ -z "${ENVOY_SRCDIR}" ] && export ENVOY_SRCDIR=/source [ -z "${ENVOY_BUILD_TARGET}" ] && export ENVOY_BUILD_TARGET=//source/exe:envoy-static [ -z "${ENVOY_BUILD_DEBUG_INFORMATION}" ] && export ENVOY_BUILD_DEBUG_INFORMATION=//source/exe:envoy-static.dwp +[ -z "${ENVOY_CONTRIB_BUILD_TARGET}" ] && export ENVOY_CONTRIB_BUILD_TARGET=//contrib/exe:envoy-static +[ -z "${ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION}" ] && export ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION=//contrib/exe:envoy-static.dwp [ -z "${ENVOY_BUILD_ARCH}" ] && { ENVOY_BUILD_ARCH=$(uname -m) export ENVOY_BUILD_ARCH diff --git a/ci/do_ci.sh b/ci/do_ci.sh index b4a4ea3c2542c..f4196156be972 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -62,47 +62,38 @@ function bazel_with_collection() { run_process_test_result } -function cp_binary_for_outside_access() { - DELIVERY_LOCATION="$1" - cp -f \ - bazel-bin/"${ENVOY_BIN}" \ - "${ENVOY_DELIVERY_DIR}"/"${DELIVERY_LOCATION}" -} - -function cp_debug_info_for_outside_access() { - DELIVERY_LOCATION="$1" - cp -f \ - bazel-bin/"${ENVOY_BIN}".dwp \ - "${ENVOY_DELIVERY_DIR}"/"${DELIVERY_LOCATION}".dwp -} - - function cp_binary_for_image_build() { + local BINARY_TYPE="$1" + local COMPILE_TYPE="$2" + local EXE_NAME="$3" + # TODO(mattklein123): Replace this with caching and a different job which creates images. local BASE_TARGET_DIR="${ENVOY_SRCDIR}${BUILD_ARCH_DIR}" + local TARGET_DIR=build_"${EXE_NAME}"_"${BINARY_TYPE}" + local FINAL_DELIVERY_DIR="${ENVOY_DELIVERY_DIR}"/"${EXE_NAME}" + echo "Copying binary for image build..." - COMPILE_TYPE="$2" - mkdir -p "${BASE_TARGET_DIR}"/build_"$1" - cp -f "${ENVOY_DELIVERY_DIR}"/envoy "${BASE_TARGET_DIR}"/build_"$1" + mkdir -p "${BASE_TARGET_DIR}"/"${TARGET_DIR}" + cp -f "${FINAL_DELIVERY_DIR}"/envoy "${BASE_TARGET_DIR}"/"${TARGET_DIR}" # Copy the su-exec utility binary into the image - cp -f bazel-bin/external/com_github_ncopa_suexec/su-exec "${BASE_TARGET_DIR}"/build_"$1" + cp -f bazel-bin/external/com_github_ncopa_suexec/su-exec "${BASE_TARGET_DIR}"/"${TARGET_DIR}" if [[ "${COMPILE_TYPE}" == "dbg" || "${COMPILE_TYPE}" == "opt" ]]; then - cp -f "${ENVOY_DELIVERY_DIR}"/envoy.dwp "${BASE_TARGET_DIR}"/build_"$1" + cp -f "${FINAL_DELIVERY_DIR}"/envoy.dwp "${BASE_TARGET_DIR}"/"${TARGET_DIR}" fi - mkdir -p "${BASE_TARGET_DIR}"/build_"$1"_stripped - strip "${ENVOY_DELIVERY_DIR}"/envoy -o "${BASE_TARGET_DIR}"/build_"$1"_stripped/envoy + mkdir -p "${BASE_TARGET_DIR}"/"${TARGET_DIR}"_stripped + strip "${FINAL_DELIVERY_DIR}"/envoy -o "${BASE_TARGET_DIR}"/"${TARGET_DIR}"_stripped/envoy # Copy for azp which doesn't preserve permissions, creating a tar archive - tar czf "${ENVOY_BUILD_DIR}"/envoy_binary.tar.gz -C "${BASE_TARGET_DIR}" build_"$1" build_"$1"_stripped + tar czf "${ENVOY_BUILD_DIR}"/"${EXE_NAME}"_binary.tar.gz -C "${BASE_TARGET_DIR}" "${TARGET_DIR}" "${TARGET_DIR}"_stripped # Remove binaries to save space, only if BUILD_REASON exists (running in AZP) [[ -z "${BUILD_REASON}" ]] || \ - rm -rf "${BASE_TARGET_DIR}"/build_"$1" "${BASE_TARGET_DIR}"/build_"$1"_stripped "${ENVOY_DELIVERY_DIR}"/envoy{,.dwp} \ + rm -rf "${BASE_TARGET_DIR:?}"/"${TARGET_DIR}" "${BASE_TARGET_DIR:?}"/"${TARGET_DIR}"_stripped "${FINAL_DELIVERY_DIR:?}"/envoy{,.dwp} \ bazel-bin/"${ENVOY_BIN}"{,.dwp} } function bazel_binary_build() { - BINARY_TYPE="$1" + local BINARY_TYPE="$1" if [[ "${BINARY_TYPE}" == "release" ]]; then COMPILE_TYPE="opt" elif [[ "${BINARY_TYPE}" == "debug" ]]; then @@ -116,31 +107,44 @@ function bazel_binary_build() { COMPILE_TYPE="fastbuild" fi - echo "Building..." - ENVOY_BIN=$(echo "${ENVOY_BUILD_TARGET}" | sed -e 's#^@\([^/]*\)/#external/\1#;s#^//##;s#:#/#') + local BUILD_TARGET="$2" + local BUILD_DEBUG_INFORMATION="$3" + local EXE_NAME="$4" + local FINAL_DELIVERY_DIR="${ENVOY_DELIVERY_DIR}"/"${EXE_NAME}" + mkdir -p "${FINAL_DELIVERY_DIR}" + + echo "Building (type=${BINARY_TYPE} target=${BUILD_TARGET} debug=${BUILD_DEBUG_INFORMATION} name=${EXE_NAME})..." + ENVOY_BIN=$(echo "${BUILD_TARGET}" | sed -e 's#^@\([^/]*\)/#external/\1#;s#^//##;s#:#/#') # This is a workaround for https://github.com/bazelbuild/bazel/issues/11834 [[ -n "${ENVOY_RBE}" ]] && rm -rf bazel-bin/"${ENVOY_BIN}"* - bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c "${COMPILE_TYPE}" "${ENVOY_BUILD_TARGET}" ${CONFIG_ARGS} + bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c "${COMPILE_TYPE}" "${BUILD_TARGET}" ${CONFIG_ARGS} collect_build_profile "${BINARY_TYPE}"_build # Copy the built envoy binary somewhere that we can access outside of the # container. - cp_binary_for_outside_access envoy + cp -f bazel-bin/"${ENVOY_BIN}" "${FINAL_DELIVERY_DIR}"/envoy if [[ "${COMPILE_TYPE}" == "dbg" || "${COMPILE_TYPE}" == "opt" ]]; then # Generate dwp file for debugging since we used split DWARF to reduce binary # size - bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c "${COMPILE_TYPE}" "${ENVOY_BUILD_DEBUG_INFORMATION}" ${CONFIG_ARGS} + bazel build "${BAZEL_BUILD_OPTIONS[@]}" -c "${COMPILE_TYPE}" "${BUILD_DEBUG_INFORMATION}" ${CONFIG_ARGS} # Copy the debug information - cp_debug_info_for_outside_access envoy + cp -f bazel-bin/"${ENVOY_BIN}".dwp "${FINAL_DELIVERY_DIR}"/envoy.dwp fi # Build su-exec utility bazel build external:su-exec - cp_binary_for_image_build "${BINARY_TYPE}" "${COMPILE_TYPE}" + cp_binary_for_image_build "${BINARY_TYPE}" "${COMPILE_TYPE}" "${EXE_NAME}" +} + +function bazel_envoy_binary_build() { + bazel_binary_build "$1" "${ENVOY_BUILD_TARGET}" "${ENVOY_BUILD_DEBUG_INFORMATION}" envoy +} +function bazel_contrib_binary_build() { + bazel_binary_build "$1" "${ENVOY_CONTRIB_BUILD_TARGET}" "${ENVOY_CONTRIB_BUILD_DEBUG_INFORMATION}" envoy-contrib } function run_process_test_result() { @@ -184,6 +188,10 @@ if [[ $# -ge 1 ]]; then else # Coverage test will add QUICHE tests by itself. COVERAGE_TEST_TARGETS=("//test/...") + if [[ "$CI_TARGET" == "bazel.release" ]]; then + # We test contrib on release only. + COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "//contrib/...") + fi TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "@com_googlesource_quiche//:ci_tests") fi @@ -200,18 +208,22 @@ if [[ "$CI_TARGET" == "bazel.release" ]]; then echo "Testing ${TEST_TARGETS[*]} with options: ${BAZEL_BUILD_OPTIONS[*]}" bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c opt "${TEST_TARGETS[@]}" - echo "bazel release build with tests..." - bazel_binary_build release + echo "bazel release build..." + bazel_envoy_binary_build release + + echo "bazel contrib release build..." + bazel_contrib_binary_build release + exit 0 elif [[ "$CI_TARGET" == "bazel.release.server_only" ]]; then setup_clang_toolchain echo "bazel release build..." - bazel_binary_build release + bazel_envoy_binary_build release exit 0 elif [[ "$CI_TARGET" == "bazel.sizeopt.server_only" ]]; then setup_clang_toolchain echo "bazel size optimized build..." - bazel_binary_build sizeopt + bazel_envoy_binary_build sizeopt exit 0 elif [[ "$CI_TARGET" == "bazel.sizeopt" ]]; then setup_clang_toolchain @@ -219,7 +231,7 @@ elif [[ "$CI_TARGET" == "bazel.sizeopt" ]]; then bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" --config=sizeopt "${TEST_TARGETS[@]}" echo "bazel size optimized build with tests..." - bazel_binary_build sizeopt + bazel_envoy_binary_build sizeopt exit 0 elif [[ "$CI_TARGET" == "bazel.gcc" ]]; then BAZEL_BUILD_OPTIONS+=("--test_env=HEAPCHECK=") @@ -229,7 +241,7 @@ elif [[ "$CI_TARGET" == "bazel.gcc" ]]; then bazel_with_collection test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild -- "${TEST_TARGETS[@]}" echo "bazel release build with gcc..." - bazel_binary_build fastbuild + bazel_envoy_binary_build fastbuild exit 0 elif [[ "$CI_TARGET" == "bazel.debug" ]]; then setup_clang_toolchain @@ -237,12 +249,12 @@ elif [[ "$CI_TARGET" == "bazel.debug" ]]; then bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c dbg "${TEST_TARGETS[@]}" echo "bazel debug build with tests..." - bazel_binary_build debug + bazel_envoy_binary_build debug exit 0 elif [[ "$CI_TARGET" == "bazel.debug.server_only" ]]; then setup_clang_toolchain echo "bazel debug build..." - bazel_binary_build debug + bazel_envoy_binary_build debug exit 0 elif [[ "$CI_TARGET" == "bazel.asan" ]]; then setup_clang_toolchain @@ -295,7 +307,7 @@ elif [[ "$CI_TARGET" == "bazel.dev" ]]; then # This doesn't go into CI but is available for developer convenience. echo "bazel fastbuild build with tests..." echo "Building..." - bazel_binary_build fastbuild + bazel_envoy_binary_build fastbuild echo "Testing ${TEST_TARGETS[*]}" bazel test "${BAZEL_BUILD_OPTIONS[@]}" -c fastbuild "${TEST_TARGETS[@]}" diff --git a/ci/docker_ci.sh b/ci/docker_ci.sh index 5ccf2dbd0c614..7fb99271c63cc 100755 --- a/ci/docker_ci.sh +++ b/ci/docker_ci.sh @@ -23,6 +23,7 @@ config_env() { build_platforms() { TYPE=$1 FILE_SUFFIX="${TYPE/-debug/}" + FILE_SUFFIX="${FILE_SUFFIX/-contrib/}" if is_windows; then echo "windows/amd64" @@ -36,8 +37,13 @@ build_platforms() { build_args() { TYPE=$1 FILE_SUFFIX="${TYPE/-debug/}" + FILE_SUFFIX="${FILE_SUFFIX/-contrib/}" printf ' -f ci/Dockerfile-envoy%s' "${FILE_SUFFIX}" + if [[ "${TYPE}" == *-contrib* ]]; then + printf ' --build-arg ENVOY_BINARY=envoy-contrib' + fi + if [[ "${TYPE}" == *-debug ]]; then printf ' --build-arg ENVOY_BINARY_SUFFIX=' elif [[ "${TYPE}" == "-google-vrp" ]]; then @@ -124,7 +130,7 @@ if is_windows; then BUILD_COMMAND=("build") else # "-google-vrp" must come afer "" to ensure we rebuild the local base image dependency. - BUILD_TYPES=("" "-debug" "-alpine" "-distroless" "-google-vrp") + BUILD_TYPES=("" "-debug" "-contrib" "-contrib-debug" "-alpine" "-distroless" "-google-vrp") # Configure docker-buildx tools BUILD_COMMAND=("buildx" "build") @@ -141,7 +147,7 @@ for BUILD_TYPE in "${BUILD_TYPES[@]}"; do build_images "${BUILD_TYPE}" "$image_tag" if ! is_windows; then - if [[ "$BUILD_TYPE" == "" || "$BUILD_TYPE" == "-alpine" ]]; then + if [[ "$BUILD_TYPE" == "" || "$BUILD_TYPE" == "-contrib" || "$BUILD_TYPE" == "-alpine" ]]; then # verify_examples expects the base and alpine images, and for them to be named `-dev` dev_image="envoyproxy/envoy${BUILD_TYPE}-dev:latest" docker tag "$image_tag" "$dev_image" diff --git a/contrib/BUILD b/contrib/BUILD new file mode 100644 index 0000000000000..aa0691c6142a8 --- /dev/null +++ b/contrib/BUILD @@ -0,0 +1,6 @@ +licenses(["notice"]) # Apache 2 + +exports_files([ + "extensions_metadata.yaml", + "contrib_build_config.bzl", +]) diff --git a/contrib/all_contrib_extensions.bzl b/contrib/all_contrib_extensions.bzl new file mode 100644 index 0000000000000..5a450825fd033 --- /dev/null +++ b/contrib/all_contrib_extensions.bzl @@ -0,0 +1,4 @@ +load(":contrib_build_config.bzl", "CONTRIB_EXTENSIONS") + +def envoy_all_contrib_extensions(): + return [v + "_envoy_extension" for v in CONTRIB_EXTENSIONS.values()] diff --git a/contrib/contrib_build_config.bzl b/contrib/contrib_build_config.bzl new file mode 100644 index 0000000000000..9c51607746a12 --- /dev/null +++ b/contrib/contrib_build_config.bzl @@ -0,0 +1,8 @@ +# See bazel/README.md for details on how this system works. +CONTRIB_EXTENSIONS = { + # + # HTTP filters + # + + "envoy.filters.http.squash": "//contrib/squash/filters/http/source:config", +} diff --git a/contrib/exe/BUILD b/contrib/exe/BUILD new file mode 100644 index 0000000000000..778ceb7dc7b45 --- /dev/null +++ b/contrib/exe/BUILD @@ -0,0 +1,24 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_binary", + "envoy_contrib_package", +) +load( + "//contrib:all_contrib_extensions.bzl", + "envoy_all_contrib_extensions", +) + +licenses(["notice"]) # Apache 2 + +envoy_contrib_package() + +alias( + name = "envoy", + actual = ":envoy-static", +) + +envoy_cc_binary( + name = "envoy-static", + stamped = True, + deps = ["//source/exe:envoy_main_entry_lib"] + envoy_all_contrib_extensions(), +) diff --git a/contrib/extensions_metadata.yaml b/contrib/extensions_metadata.yaml new file mode 100644 index 0000000000000..d841fc7d5f737 --- /dev/null +++ b/contrib/extensions_metadata.yaml @@ -0,0 +1,5 @@ +envoy.filters.http.squash: + categories: + - envoy.filters.http + security_posture: requires_trusted_downstream_and_upstream + status: stable diff --git a/source/extensions/filters/http/squash/BUILD b/contrib/squash/filters/http/source/BUILD similarity index 78% rename from source/extensions/filters/http/squash/BUILD rename to contrib/squash/filters/http/source/BUILD index 65831764d4c7e..897149d5137fc 100644 --- a/source/extensions/filters/http/squash/BUILD +++ b/contrib/squash/filters/http/source/BUILD @@ -1,8 +1,8 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_cc_extension", + "envoy_cc_contrib_extension", "envoy_cc_library", - "envoy_extension_package", + "envoy_contrib_package", ) licenses(["notice"]) # Apache 2 @@ -10,7 +10,7 @@ licenses(["notice"]) # Apache 2 # L7 HTTP filter that implements the Squash microservice debugger # Public docs: docs/root/configuration/http_filters/squash_filter.rst -envoy_extension_package() +envoy_contrib_package() envoy_cc_library( name = "squash_filter_lib", @@ -29,19 +29,19 @@ envoy_cc_library( "//source/common/http:utility_lib", "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) -envoy_cc_extension( +envoy_cc_contrib_extension( name = "config", srcs = ["config.cc"], hdrs = ["config.h"], deps = [ + ":squash_filter_lib", "//envoy/registry", "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", - "//source/extensions/filters/http/squash:squash_filter_lib", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/squash/config.cc b/contrib/squash/filters/http/source/config.cc similarity index 80% rename from source/extensions/filters/http/squash/config.cc rename to contrib/squash/filters/http/source/config.cc index d2420a6bb8381..cec853979f2e6 100644 --- a/source/extensions/filters/http/squash/config.cc +++ b/contrib/squash/filters/http/source/config.cc @@ -1,12 +1,13 @@ -#include "source/extensions/filters/http/squash/config.h" +#include "contrib/squash/filters/http/source/config.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" #include "envoy/registry/registry.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" -#include "source/extensions/filters/http/squash/squash_filter.h" + +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" +#include "contrib/squash/filters/http/source/squash_filter.h" namespace Envoy { namespace Extensions { diff --git a/source/extensions/filters/http/squash/config.h b/contrib/squash/filters/http/source/config.h similarity index 84% rename from source/extensions/filters/http/squash/config.h rename to contrib/squash/filters/http/source/config.h index c1514fe61ab97..feff75eaf8b0e 100644 --- a/source/extensions/filters/http/squash/config.h +++ b/contrib/squash/filters/http/source/config.h @@ -1,10 +1,10 @@ #pragma once -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" - #include "source/extensions/filters/http/common/factory_base.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" + namespace Envoy { namespace Extensions { namespace HttpFilters { diff --git a/source/extensions/filters/http/squash/squash_filter.cc b/contrib/squash/filters/http/source/squash_filter.cc similarity index 98% rename from source/extensions/filters/http/squash/squash_filter.cc rename to contrib/squash/filters/http/source/squash_filter.cc index e2db20a9f2d2e..41fc37e5f7613 100644 --- a/source/extensions/filters/http/squash/squash_filter.cc +++ b/contrib/squash/filters/http/source/squash_filter.cc @@ -1,8 +1,7 @@ -#include "source/extensions/filters/http/squash/squash_filter.h" +#include "contrib/squash/filters/http/source/squash_filter.h" #include -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" #include "envoy/http/codes.h" #include "source/common/common/empty_string.h" @@ -16,6 +15,7 @@ #include "source/common/protobuf/utility.h" #include "absl/container/fixed_array.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" namespace Envoy { namespace Extensions { @@ -91,7 +91,7 @@ void SquashFilterConfig::updateTemplateInValue(ProtobufWkt::Value& curvalue) { To interpolate an environment variable named ENV, add '{{ ENV }}' (without the quotes, with the spaces) to the template string. - See api/envoy/extensions/filters/http/squash/v3/squash.proto for the motivation on why this is + See api/envoy/extensions/squash/filters/http/v3/squash.proto for the motivation on why this is needed. */ std::string SquashFilterConfig::replaceEnv(const std::string& attachment_template) { diff --git a/source/extensions/filters/http/squash/squash_filter.h b/contrib/squash/filters/http/source/squash_filter.h similarity index 98% rename from source/extensions/filters/http/squash/squash_filter.h rename to contrib/squash/filters/http/source/squash_filter.h index 6786a6117a741..df1688210ac4b 100644 --- a/source/extensions/filters/http/squash/squash_filter.h +++ b/contrib/squash/filters/http/source/squash_filter.h @@ -2,7 +2,6 @@ #include -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" #include "envoy/http/async_client.h" #include "envoy/http/filter.h" #include "envoy/json/json_object.h" @@ -12,6 +11,7 @@ #include "source/common/protobuf/protobuf.h" #include "absl/types/optional.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/squash/BUILD b/contrib/squash/filters/http/test/BUILD similarity index 56% rename from test/extensions/filters/http/squash/BUILD rename to contrib/squash/filters/http/test/BUILD index f03de9e17e6d3..a2a9b49f4c9ad 100644 --- a/test/extensions/filters/http/squash/BUILD +++ b/contrib/squash/filters/http/test/BUILD @@ -1,52 +1,46 @@ load( "//bazel:envoy_build_system.bzl", - "envoy_package", -) -load( - "//test/extensions:extensions_build_system.bzl", - "envoy_extension_cc_test", + "envoy_cc_test", + "envoy_contrib_package", ) licenses(["notice"]) # Apache 2 -envoy_package() +envoy_contrib_package() -envoy_extension_cc_test( +envoy_cc_test( name = "squash_filter_test", srcs = ["squash_filter_test.cc"], - extension_names = ["envoy.filters.http.squash"], deps = [ + "//contrib/squash/filters/http/source:squash_filter_lib", "//envoy/event:dispatcher_interface", "//source/common/http:header_map_lib", "//source/common/stats:stats_lib", - "//source/extensions/filters/http/squash:squash_filter_lib", "//test/mocks/http:http_mocks", "//test/mocks/server:factory_context_mocks", "//test/mocks/upstream:cluster_manager_mocks", "//test/test_common:environment_lib", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) -envoy_extension_cc_test( +envoy_cc_test( name = "squash_filter_integration_test", srcs = ["squash_filter_integration_test.cc"], - extension_names = ["envoy.filters.http.squash"], deps = [ - "//source/extensions/filters/http/squash:config", + "//contrib/squash/filters/http/source:config", "//test/integration:http_integration_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", ], ) -envoy_extension_cc_test( +envoy_cc_test( name = "config_test", srcs = ["config_test.cc"], - extension_names = ["envoy.filters.http.squash"], deps = [ - "//source/extensions/filters/http/squash:config", + "//contrib/squash/filters/http/source:config", "//test/mocks/server:factory_context_mocks", "//test/test_common:utility_lib", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", + "@envoy_api//contrib/envoy/extensions/filters/http/squash/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/filters/http/squash/config_test.cc b/contrib/squash/filters/http/test/config_test.cc similarity index 88% rename from test/extensions/filters/http/squash/config_test.cc rename to contrib/squash/filters/http/test/config_test.cc index 8964bb665308c..613dc5b67e9ad 100644 --- a/test/extensions/filters/http/squash/config_test.cc +++ b/contrib/squash/filters/http/test/config_test.cc @@ -1,11 +1,9 @@ -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" - -#include "source/extensions/filters/http/squash/config.h" - #include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.validate.h" +#include "contrib/squash/filters/http/source/config.h" #include "gmock/gmock.h" #include "gtest/gtest.h" diff --git a/test/extensions/filters/http/squash/squash_filter_integration_test.cc b/contrib/squash/filters/http/test/squash_filter_integration_test.cc similarity index 100% rename from test/extensions/filters/http/squash/squash_filter_integration_test.cc rename to contrib/squash/filters/http/test/squash_filter_integration_test.cc diff --git a/test/extensions/filters/http/squash/squash_filter_test.cc b/contrib/squash/filters/http/test/squash_filter_test.cc similarity index 99% rename from test/extensions/filters/http/squash/squash_filter_test.cc rename to contrib/squash/filters/http/test/squash_filter_test.cc index be5d8d852bfe3..756e83e402fb5 100644 --- a/test/extensions/filters/http/squash/squash_filter_test.cc +++ b/contrib/squash/filters/http/test/squash_filter_test.cc @@ -2,18 +2,16 @@ #include #include -#include "envoy/common/scope_tracker.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" - #include "source/common/http/message_impl.h" #include "source/common/protobuf/protobuf.h" -#include "source/extensions/filters/http/squash/squash_filter.h" #include "test/mocks/server/factory_context.h" #include "test/mocks/upstream/cluster_manager.h" #include "test/test_common/environment.h" #include "test/test_common/utility.h" +#include "contrib/envoy/extensions/filters/http/squash/v3/squash.pb.h" +#include "contrib/squash/filters/http/source/squash_filter.h" #include "fmt/format.h" #include "gmock/gmock.h" #include "gtest/gtest.h" diff --git a/docs/BUILD b/docs/BUILD index e2a0c33e0114f..51f3e2b4a1cd3 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -115,11 +115,15 @@ pkg_tar( genrule( name = "extensions_security_rst", - srcs = ["//source/extensions:extensions_metadata.yaml"], + srcs = [ + "//source/extensions:extensions_metadata.yaml", + "//contrib:extensions_metadata.yaml", + ], outs = ["extensions_security_rst.tar"], cmd = """ $(location //tools/docs:generate_extensions_security_rst) \\ - $(location //source/extensions:extensions_metadata.yaml) $@ + $(location //source/extensions:extensions_metadata.yaml) \\ + $(location //contrib:extensions_metadata.yaml) $@ """, tools = ["//tools/docs:generate_extensions_security_rst"], ) diff --git a/docs/root/intro/arch_overview/security/threat_model.rst b/docs/root/intro/arch_overview/security/threat_model.rst index 66b906f1cb281..93aa6530b545e 100644 --- a/docs/root/intro/arch_overview/security/threat_model.rst +++ b/docs/root/intro/arch_overview/security/threat_model.rst @@ -83,6 +83,11 @@ and do not qualify for treatment under the threat model below. As a consequence, with this model in mind. Security issues related to core code will usually trigger the security release process as described in this document. +.. note:: + + :ref:`contrib ` extensions are noted below and are not officially covered by + the threat model or the Envoy security team. All indications below are best effort. + The following extensions are intended to be hardened against untrusted downstream and upstreams: .. include:: secpos_robust_to_untrusted_downstream_and_upstream.rst diff --git a/docs/root/start/install.rst b/docs/root/start/install.rst index 79a651c12f3b5..30d6a17c03503 100644 --- a/docs/root/start/install.rst +++ b/docs/root/start/install.rst @@ -110,6 +110,20 @@ The following commands will pull and show the Envoy version of current images. $ docker pull envoyproxy/|envoy_distroless_docker_image| $ docker run --rm envoyproxy/|envoy_distroless_docker_image| --version +.. _install_contrib: + +Contrib images +~~~~~~~~~~~~~~ + +As described in `this document `_, +the Envoy project allows extensions to enter the repository as "contrib" extensions. The requirements +for such extensions are lower, and as such they are only available by default in special images. +The `envoyproxy/envoy-contrib `_ image +contains all contrib extensions on top of an Ubuntu base. The +`envoyproxy/envoy-contrib-debug `_ +image contains all contrib extensions on top of an Ubuntu base as well as debug symbols. Throughout +the documentation, extensions are clearly marked as being a contrib extension or a core extension. + .. _install_binaries: Pre-built Envoy Docker images @@ -140,6 +154,12 @@ The following table shows the available Docker images - |DOCKER_IMAGE_TAG_NAME| - - + * - `envoyproxy/envoy-contrib `_ + - Release :ref:`contrib ` binary with symbols stripped on top of an Ubuntu Bionic base. + - |DOCKER_IMAGE_TAG_NAME| + - |DOCKER_IMAGE_TAG_NAME| + - + - * - `envoyproxy/envoy-distroless `_ - Release binary with symbols stripped on top of a distroless base. - |DOCKER_IMAGE_TAG_NAME| @@ -164,12 +184,24 @@ The following table shows the available Docker images - |DOCKER_IMAGE_TAG_NAME| - - + * - `envoyproxy/envoy-contrib-debug `_ + - Release :ref:`contrib ` binary with debug symbols on top of an Ubuntu Bionic base. + - |DOCKER_IMAGE_TAG_NAME| + - |DOCKER_IMAGE_TAG_NAME| + - + - * - `envoyproxy/envoy-dev `_ - Release binary with symbols stripped on top of an Ubuntu Bionic base. - - - latest - latest + * - `envoyproxy/envoy-contrib-dev `_ + - Release :ref:`contrib ` binary with symbols stripped on top of an Ubuntu Bionic base. + - + - + - latest + - latest * - `envoyproxy/envoy-distroless-dev `_ - Release binary with symbols stripped on top of a distroless base. - @@ -188,6 +220,12 @@ The following table shows the available Docker images - - latest - latest + * - `envoyproxy/envoy-contrib-debug-dev `_ + - Release :ref:`contrib ` binary with debug symbols on top of an Ubuntu Bionic base. + - + - + - latest + - latest * - `envoyproxy/envoy-windows-dev `_ - Release binary with symbols stripped on top of a Windows Server 1809 base. Includes build tools. - diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 7f6b88838cd76..1fd3573062d2d 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -5,6 +5,8 @@ Incompatible Behavior Changes ----------------------------- *Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* +* contrib: the :ref:`squash filter ` has been moved to + :ref:`contrib images `. * ext_authz: fixed skipping authentication when returning either a direct response or a redirect. This behavior can be temporarily reverted by setting the ``envoy.reloadable_features.http_ext_authz_do_not_skip_direct_response_and_redirect`` runtime guard to false. Minor Behavior Changes @@ -62,6 +64,7 @@ New Features ------------ * access_log: added :ref:`METADATA` token to handle all types of metadata (DYNAMIC, CLUSTER, ROUTE). * bootstrap: added :ref:`inline_headers ` in the bootstrap to make custom inline headers bootstrap configurable. +* contrib: added new :ref:`contrib images ` which contain contrib extensions. * grpc reverse bridge: added a new :ref:`option ` to support streaming response bodies when withholding gRPC frames from the upstream. * http: added :ref:`string_match ` in the header matcher. * http: added support for :ref:`max_requests_per_connection ` for both upstream and downstream connections. diff --git a/generated_api_shadow/BUILD b/generated_api_shadow/BUILD index 24e0ea0de018a..044bc7d137a88 100644 --- a/generated_api_shadow/BUILD +++ b/generated_api_shadow/BUILD @@ -57,6 +57,7 @@ proto_library( name = "v3_protos", visibility = ["//visibility:public"], deps = [ + "//contrib/envoy/extensions/filters/http/squash/v3:pkg", "//envoy/admin/v3:pkg", "//envoy/config/accesslog/v3:pkg", "//envoy/config/bootstrap/v3:pkg", @@ -143,7 +144,6 @@ proto_library( "//envoy/extensions/filters/http/rbac/v3:pkg", "//envoy/extensions/filters/http/router/v3:pkg", "//envoy/extensions/filters/http/set_metadata/v3:pkg", - "//envoy/extensions/filters/http/squash/v3:pkg", "//envoy/extensions/filters/http/tap/v3:pkg", "//envoy/extensions/filters/http/wasm/v3:pkg", "//envoy/extensions/filters/listener/http_inspector/v3:pkg", diff --git a/generated_api_shadow/envoy/extensions/filters/http/squash/v3/BUILD b/generated_api_shadow/contrib/envoy/extensions/filters/http/squash/v3/BUILD similarity index 100% rename from generated_api_shadow/envoy/extensions/filters/http/squash/v3/BUILD rename to generated_api_shadow/contrib/envoy/extensions/filters/http/squash/v3/BUILD diff --git a/generated_api_shadow/envoy/extensions/filters/http/squash/v3/squash.proto b/generated_api_shadow/contrib/envoy/extensions/filters/http/squash/v3/squash.proto similarity index 100% rename from generated_api_shadow/envoy/extensions/filters/http/squash/v3/squash.proto rename to generated_api_shadow/contrib/envoy/extensions/filters/http/squash/v3/squash.proto diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 8fbb6ffcee499..3f8dfeb00f0bc 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -103,7 +103,6 @@ EXTENSIONS = { "envoy.filters.http.rbac": "//source/extensions/filters/http/rbac:config", "envoy.filters.http.router": "//source/extensions/filters/http/router:config", "envoy.filters.http.set_metadata": "//source/extensions/filters/http/set_metadata:config", - "envoy.filters.http.squash": "//source/extensions/filters/http/squash:config", "envoy.filters.http.tap": "//source/extensions/filters/http/tap:config", "envoy.filters.http.wasm": "//source/extensions/filters/http/wasm:config", diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 44e8389f1eb9b..aaef9bda41909 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -263,11 +263,6 @@ envoy.filters.http.set_metadata: - envoy.filters.http security_posture: robust_to_untrusted_downstream_and_upstream status: stable -envoy.filters.http.squash: - categories: - - envoy.filters.http - security_posture: requires_trusted_downstream_and_upstream - status: stable envoy.filters.http.tap: categories: - envoy.filters.http diff --git a/test/extensions/filters/http/common/fuzz/BUILD b/test/extensions/filters/http/common/fuzz/BUILD index ad1be79cf3413..04bc4579103fe 100644 --- a/test/extensions/filters/http/common/fuzz/BUILD +++ b/test/extensions/filters/http/common/fuzz/BUILD @@ -57,7 +57,6 @@ envoy_cc_test_library( "//test/test_common:test_runtime_lib", "@envoy_api//envoy/extensions/filters/http/grpc_json_transcoder/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/jwt_authn/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/http/squash/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/tap/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc index ce6720a2d2bf2..b1e2675f8b60c 100644 --- a/test/extensions/filters/http/common/fuzz/uber_per_filter.cc +++ b/test/extensions/filters/http/common/fuzz/uber_per_filter.cc @@ -1,6 +1,5 @@ #include "envoy/extensions/filters/http/grpc_json_transcoder/v3/transcoder.pb.h" #include "envoy/extensions/filters/http/jwt_authn/v3/config.pb.h" -#include "envoy/extensions/filters/http/squash/v3/squash.pb.h" #include "envoy/extensions/filters/http/tap/v3/tap.pb.h" #include "source/common/tracing/http_tracer_impl.h" @@ -74,17 +73,6 @@ void UberFilterFuzzer::guideAnyProtoType(test::fuzz::HttpData* mutable_data, uin mutable_any->set_type_url(type_url); } -void cleanAttachmentTemplate(Protobuf::Message* message) { - envoy::extensions::filters::http::squash::v3::Squash& config = - dynamic_cast(*message); - std::string json; - Protobuf::util::JsonPrintOptions json_options; - if (!Protobuf::util::MessageToJsonString(config.attachment_template(), &json, json_options) - .ok()) { - config.clear_attachment_template(); - } -} - void cleanTapConfig(Protobuf::Message* message) { envoy::extensions::filters::http::tap::v3::Tap& config = dynamic_cast(*message); @@ -111,8 +99,6 @@ void UberFilterFuzzer::cleanFuzzedConfig(absl::string_view filter_name, if (filter_name == "envoy.filters.http.grpc_json_transcoder") { // Add a valid service proto descriptor. addBookstoreProtoDescriptor(message); - } else if (name == "envoy.filters.http.squash") { - cleanAttachmentTemplate(message); } else if (name == "envoy.filters.http.tap") { // TapDS oneof field and OutputSinkType StreamingGrpc not implemented cleanTapConfig(message); diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py index cddeafcc42a3a..2b65d6b904db6 100755 --- a/tools/code_format/check_format.py +++ b/tools/code_format/check_format.py @@ -23,6 +23,7 @@ "./bazel-", "./.cache", "./source/extensions/extensions_build_config.bzl", + "./contrib/contrib_build_config.bzl", "./bazel/toolchains/configs/", "./tools/testdata/check_format/", "./tools/pyformat/", @@ -97,8 +98,8 @@ "./source/common/common/regex.cc", "./source/common/stats/tag_extractor_impl.h", "./source/common/stats/tag_extractor_impl.cc", "./source/common/formatter/substitution_formatter.cc", - "./source/extensions/filters/http/squash/squash_filter.h", - "./source/extensions/filters/http/squash/squash_filter.cc", "./source/server/admin/utils.h", + "./contrib/squash/filters/http/source/squash_filter.h", + "./contrib/squash/filters/http/source/squash_filter.cc", "./source/server/admin/utils.h", "./source/server/admin/utils.cc", "./source/server/admin/stats_handler.h", "./source/server/admin/stats_handler.cc", "./source/server/admin/prometheus_stats.h", "./source/server/admin/prometheus_stats.cc", "./tools/clang_tools/api_booster/main.cc", @@ -176,6 +177,7 @@ HISTOGRAM_SI_SUFFIX_REGEX = re.compile(r"(?<=HISTOGRAM\()[a-zA-Z0-9_]+_(b|kb|mb|ns|us|ms|s)(?=,)") TEST_NAME_STARTING_LOWER_CASE_REGEX = re.compile(r"TEST(_.\(.*,\s|\()[a-z].*\)\s\{") EXTENSIONS_CODEOWNERS_REGEX = re.compile(r'.*(extensions[^@]*\s+)(@.*)') +CONTRIB_CODEOWNERS_REGEX = re.compile(r'(/contrib/\w+/\s+)(@.*)') COMMENT_REGEX = re.compile(r"//|\*") DURATION_VALUE_REGEX = re.compile(r'\b[Dd]uration\(([0-9.]+)') PROTO_VALIDATION_STRING = re.compile(r'\bmin_bytes\b') @@ -1086,13 +1088,20 @@ def check_format_visitor(self, arg, dir_name, names): # Sanity check CODEOWNERS. This doesn't need to be done in a multi-threaded # manner as it is a small and limited list. source_prefix = './source/' - full_prefix = './source/extensions/' + core_extensions_full_prefix = './source/extensions/' # Check to see if this directory is a subdir under /source/extensions # Also ignore top level directories under /source/extensions since we don't # need owners for source/extensions/access_loggers etc, just the subdirectories. - if dir_name.startswith(full_prefix) and '/' in dir_name[len(full_prefix):]: + if dir_name.startswith( + core_extensions_full_prefix) and '/' in dir_name[len(core_extensions_full_prefix):]: self.check_owners(dir_name[len(source_prefix):], owned_directories, error_messages) + # For contrib extensions we track ownership at the top level only. + contrib_prefix = './contrib/' + if dir_name.startswith(contrib_prefix): + top_level = pathlib.PurePath('/', *pathlib.PurePath(dir_name).parts[:2], '/') + self.check_owners(str(top_level), owned_directories, error_messages) + for file_name in names: if dir_name.startswith("./api") and self.is_starlark_file(file_name): result = pool.apply_async( @@ -1214,6 +1223,15 @@ def owned_directories(error_messages): "Extensions require at least one maintainer OWNER:\n" " {}".format(line)) + m = CONTRIB_CODEOWNERS_REGEX.search(line) + if m is not None and not line.startswith('#'): + owned.append(m.group(1).strip()) + owners = re.findall('@\S+', m.group(2).strip()) + if len(owners) < 2: + error_messages.append( + "Contrib extensions require at least 2 owners in CODEOWNERS:\n" + " {}".format(line)) + return owned except IOError: return [] # for the check format tests. diff --git a/tools/code_format/envoy_build_fixer.py b/tools/code_format/envoy_build_fixer.py index cfdbf59d24a90..7de2a307889c0 100755 --- a/tools/code_format/envoy_build_fixer.py +++ b/tools/code_format/envoy_build_fixer.py @@ -34,6 +34,7 @@ # Match a load() statement for the envoy_package macros. PACKAGE_LOAD_BLOCK_REGEX = re.compile('("envoy_package".*?\)\n)', re.DOTALL) EXTENSION_PACKAGE_LOAD_BLOCK_REGEX = re.compile('("envoy_extension_package".*?\)\n)', re.DOTALL) +CONTRIB_PACKAGE_LOAD_BLOCK_REGEX = re.compile('("envoy_contrib_package".*?\)\n)', re.DOTALL) # Match Buildozer 'print' output. Example of Buildozer print output: # cc_library json_transcoder_filter_lib [json_transcoder_filter.cc] (missing) (missing) @@ -41,7 +42,7 @@ '\s*([\w_]+)\s+([\w_]+)\s+[(\[](.*?)[)\]]\s+[(\[](.*?)[)\]]\s+[(\[](.*?)[)\]]') # Match API header include in Envoy source file? -API_INCLUDE_REGEX = re.compile('#include "(envoy/.*)/[^/]+\.pb\.(validate\.)?h"') +API_INCLUDE_REGEX = re.compile('#include "(contrib/envoy/.*|envoy/.*)/[^/]+\.pb\.(validate\.)?h"') class EnvoyBuildFixerError(Exception): @@ -79,6 +80,10 @@ def fix_package_and_license(path, contents): regex_to_use = EXTENSION_PACKAGE_LOAD_BLOCK_REGEX package_string = 'envoy_extension_package' + if 'contrib/' in path: + regex_to_use = CONTRIB_PACKAGE_LOAD_BLOCK_REGEX + package_string = 'envoy_contrib_package' + # Ensure we have an envoy_package import load if this is a real Envoy package. We also allow # the prefix to be overridden if envoy is included in a larger workspace. if re.search(ENVOY_RULE_REGEX, contents): diff --git a/tools/docs/BUILD b/tools/docs/BUILD index 39049acc40cfa..56892499d24d5 100644 --- a/tools/docs/BUILD +++ b/tools/docs/BUILD @@ -10,7 +10,6 @@ envoy_package() py_binary( name = "generate_extensions_security_rst", srcs = ["generate_extensions_security_rst.py"], - data = ["//source/extensions:extensions_metadata.yaml"], deps = [ "//tools/base:utils", ], diff --git a/tools/docs/generate_api_rst.py b/tools/docs/generate_api_rst.py index 2076b50d44277..e5539332de1fb 100644 --- a/tools/docs/generate_api_rst.py +++ b/tools/docs/generate_api_rst.py @@ -4,6 +4,24 @@ import tarfile +def include_package(envoy_api_protos, rst_file_path, prefix): + # `envoy_api_rst_files` is a list of file paths for .proto.rst files + # generated by protodoc + # + # we are only interested in the proto files generated for envoy protos, + # not for non-envoy dependencies + if ("pkg/" + prefix) not in rst_file_path: + return None + # derive the "canonical" path from the filepath + canonical = f"{rst_file_path.split('pkg/' + prefix)[1]}" + + # we are only interested in the actual v3 protos, not their dependencies + if (prefix + canonical) not in envoy_api_protos: + return None + + return canonical + + def main(): proto_srcs = sys.argv[1] envoy_api_rst_files = sys.argv[1:-1] @@ -24,19 +42,12 @@ def main(): ] for rst_file_path in envoy_api_rst_files: - # `envoy_api_rst_files` is a list of file paths for .proto.rst files - # generated by protodoc - # - # we are only interested in the proto files generated for envoy protos, - # not for non-envoy dependencies - if "pkg/envoy" not in rst_file_path: + canonical = include_package(envoy_api_protos, rst_file_path, "envoy/") + if canonical is None: + canonical = include_package(envoy_api_protos, rst_file_path, "contrib/envoy/") + if canonical is None: continue - # derive the "canonical" path from the filepath - canonical = f"{rst_file_path.split('pkg/envoy/')[1]}" - # we are only interested in the actual v3 protos, not their dependencies - if f"envoy/{canonical}" not in envoy_api_protos: - continue target = os.path.join("rst-out/api-v3", canonical) if not os.path.exists(os.path.dirname(target)): os.makedirs(os.path.dirname(target)) diff --git a/tools/docs/generate_extensions_security_rst.py b/tools/docs/generate_extensions_security_rst.py index 2dc6d8e84ee29..180e2eb247f69 100644 --- a/tools/docs/generate_extensions_security_rst.py +++ b/tools/docs/generate_extensions_security_rst.py @@ -18,16 +18,24 @@ def format_item(extension, metadata): item = '* :ref:`%s `' % (extension, extension) if metadata.get('status') == 'alpha': item += ' (alpha)' + if metadata.get('contrib', False): + item += ' (:ref:`contrib builds ` only)' return item def main(): metadata_filepath = sys.argv[1] - output_filename = sys.argv[2] + contrib_metadata_filepath = sys.argv[2] + output_filename = sys.argv[3] generated_rst_dir = os.path.dirname(output_filename) security_rst_root = os.path.join(generated_rst_dir, "intro/arch_overview/security") extension_db = utils.from_yaml(metadata_filepath) + contrib_extension_db = utils.from_yaml(contrib_metadata_filepath) + for contrib_extension in contrib_extension_db.keys(): + contrib_extension_db[contrib_extension]['contrib'] = True + extension_db.update(contrib_extension_db) + pathlib.Path(security_rst_root).mkdir(parents=True, exist_ok=True) security_postures = defaultdict(list) diff --git a/tools/extensions/BUILD b/tools/extensions/BUILD index b3f2dcbecaddb..52147a0e6b446 100644 --- a/tools/extensions/BUILD +++ b/tools/extensions/BUILD @@ -9,6 +9,8 @@ envoy_package() envoy_py_binary( name = "tools.extensions.extensions_check", data = [ + "//contrib:contrib_build_config.bzl", + "//contrib:extensions_metadata.yaml", "//source/extensions:extensions_metadata.yaml", "//source/extensions:extensions_build_config.bzl", "//test/extensions/filters/network/common/fuzz:uber_per_readfilter.cc", diff --git a/tools/extensions/extensions_check.py b/tools/extensions/extensions_check.py index 7a4b50fa52d83..8f67ad1e72a7d 100644 --- a/tools/extensions/extensions_check.py +++ b/tools/extensions/extensions_check.py @@ -14,6 +14,7 @@ from tools.base import checker, utils BUILD_CONFIG_PATH = "source/extensions/extensions_build_config.bzl" +CONTRIB_BUILD_CONFIG_PATH = "contrib/contrib_build_config.bzl" BUILTIN_EXTENSIONS = ( "envoy.request_id.uuid", "envoy.upstreams.tcp.generic", "envoy.transport_sockets.tls", @@ -70,6 +71,7 @@ FUZZ_TEST_PATH = "test/extensions/filters/network/common/fuzz/uber_per_readfilter.cc" METADATA_PATH = "source/extensions/extensions_metadata.yaml" +CONTRIB_METADATA_PATH = "contrib/extensions_metadata.yaml" class ExtensionsConfigurationError(Exception): @@ -84,25 +86,18 @@ class ExtensionsChecker(checker.Checker): @cached_property def all_extensions(self) -> set: - return set(self.configured_extensions.keys()) | set(BUILTIN_EXTENSIONS) + return set(self.configured_extensions.keys()) | set( + self.configured_contrib_extensions.keys()) | set(BUILTIN_EXTENSIONS) @cached_property def configured_extensions(self) -> dict: - # source/extensions/extensions_build_config.bzl must have a - # .bzl suffix for Starlark import, so we are forced to do this workaround. - _extensions_build_config_spec = spec_from_loader( - "extensions_build_config", - SourceFileLoader("extensions_build_config", BUILD_CONFIG_PATH)) + return ExtensionsChecker._load_build_config( + "extensions_build_config", BUILD_CONFIG_PATH, "EXTENSIONS") - if not isinstance(_extensions_build_config_spec, ModuleSpec): - raise ExtensionsConfigurationError(f"Unable to parse build config {BUILD_CONFIG_PATH}") - extensions_build_config = module_from_spec(_extensions_build_config_spec) - - if not isinstance(_extensions_build_config_spec.loader, Loader): - raise ExtensionsConfigurationError(f"Unable to parse build config {BUILD_CONFIG_PATH}") - - _extensions_build_config_spec.loader.exec_module(extensions_build_config) - return getattr(extensions_build_config, "EXTENSIONS") + @cached_property + def configured_contrib_extensions(self) -> dict: + return ExtensionsChecker._load_build_config( + "contrib_build_config", CONTRIB_BUILD_CONFIG_PATH, "CONTRIB_EXTENSIONS") @property def fuzzed_count(self) -> int: @@ -114,8 +109,10 @@ def fuzzed_count(self) -> int: @cached_property def metadata(self) -> dict: result = utils.from_yaml(METADATA_PATH) + result.update(utils.from_yaml(CONTRIB_METADATA_PATH)) if not isinstance(result, dict): - raise ExtensionsConfigurationError(f"Unable to parse configuration: {METADATA_PATH}") + raise ExtensionsConfigurationError( + f"Unable to parse metadata: {METADATA_PATH} {CONTRIB_METADATA_PATH}") return result @property @@ -126,6 +123,23 @@ def robust_to_downstream_count(self) -> int: if "network" in ext and data["security_posture"] == "robust_to_untrusted_downstream" ]) + @staticmethod + def _load_build_config(name, build_config_path, dictionary_name) -> dict: + # build configs must have a .bzl suffix for Starlark import, so we are forced to do this + # workaround. + _extensions_build_config_spec = spec_from_loader( + name, SourceFileLoader(name, build_config_path)) + + if not isinstance(_extensions_build_config_spec, ModuleSpec): + raise ExtensionsConfigurationError(f"Unable to parse build config {build_config_path}") + extensions_build_config = module_from_spec(_extensions_build_config_spec) + + if not isinstance(_extensions_build_config_spec.loader, Loader): + raise ExtensionsConfigurationError(f"Unable to parse build config {build_config_path}") + + _extensions_build_config_spec.loader.exec_module(extensions_build_config) + return getattr(extensions_build_config, dictionary_name) + def check_fuzzed(self) -> None: if self.robust_to_downstream_count == self.fuzzed_count: return diff --git a/tools/extensions/tests/test_extensions_check.py b/tools/extensions/tests/test_extensions_check.py index 12c7ef9a83814..1a194aa89b612 100644 --- a/tools/extensions/tests/test_extensions_check.py +++ b/tools/extensions/tests/test_extensions_check.py @@ -1,6 +1,6 @@ import types from importlib.machinery import ModuleSpec -from unittest.mock import patch, PropertyMock +from unittest.mock import patch, PropertyMock, call import pytest @@ -26,7 +26,7 @@ def test_extensions_checker_all_extensions(): assert ( result - == set(_configured.keys()) | set(extensions_check.BUILTIN_EXTENSIONS)) + == set(_configured.keys()) | set(extensions_check.BUILTIN_EXTENSIONS) | set(checker.configured_contrib_extensions.keys())) assert "all_extensions" in checker.__dict__ @@ -134,13 +134,13 @@ def test_extensions_metadata(patches, is_dict): checker.metadata assert ( - list(m_utils.from_yaml.call_args) - == [(extensions_check.METADATA_PATH,), {}]) + list(m_utils.from_yaml.call_args_list) + == [call(extensions_check.METADATA_PATH), call(extensions_check.CONTRIB_METADATA_PATH)]) if not is_dict: assert ( e.value.args[0] - == f'Unable to parse configuration: {extensions_check.METADATA_PATH}') + == f'Unable to parse metadata: {extensions_check.METADATA_PATH} {extensions_check.CONTRIB_METADATA_PATH}') return assert "metadata" in checker.__dict__ diff --git a/tools/gen_compilation_database.py b/tools/gen_compilation_database.py index a80fea082417a..8c13694b15d2d 100755 --- a/tools/gen_compilation_database.py +++ b/tools/gen_compilation_database.py @@ -93,6 +93,13 @@ def fix_compilation_database(args, db): parser.add_argument('--include_headers', action='store_true') parser.add_argument('--vscode', action='store_true') parser.add_argument( - 'bazel_targets', nargs='*', default=["//source/...", "//test/...", "//tools/..."]) + 'bazel_targets', + nargs='*', + default=[ + "//source/...", + "//test/...", + "//tools/...", + "//contrib/...", + ]) args = parser.parse_args() fix_compilation_database(args, generate_compilation_database(args)) diff --git a/tools/proto_format/proto_sync.py b/tools/proto_format/proto_sync.py index 52bc5710ce675..6eae2561420b1 100755 --- a/tools/proto_format/proto_sync.py +++ b/tools/proto_format/proto_sync.py @@ -41,6 +41,12 @@ 'validate/validate.proto', ] +# Each of the following contrib extensions are allowed to be in the v3 namespace. Indicate why. +CONTRIB_V3_ALLOW_LIST = [ + # Extension moved from core to contrib. + 'envoy.extensions.filters.http.squash.v3', +] + BUILD_FILE_TEMPLATE = string.Template( """# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. @@ -162,8 +168,20 @@ def get_destination_path(src): if len(matches) != 1: raise RequiresReformatError( "Expect {} has only one package declaration but has {}".format(src, len(matches))) - return pathlib.Path(get_directory_from_package( - matches[0])).joinpath(src_path.name.split('.')[0] + ".proto") + package = matches[0] + dst_path = pathlib.Path( + get_directory_from_package(package)).joinpath(src_path.name.split('.')[0] + ".proto") + # contrib API files have the standard namespace but are in a contrib folder for clarity. + # The following prepends contrib for contrib packages so we wind up with the real final path. + if 'contrib' in src: + if 'v3alpha' not in package and package not in CONTRIB_V3_ALLOW_LIST: + raise ProtoSyncError( + "contrib extension package '{}' does not use v3alpha namespace. " + "Add to CONTRIB_V3_ALLOW_LIST with an explanation if this is on purpose.".format( + package)) + + dst_path = pathlib.Path('contrib').joinpath(dst_path) + return dst_path def get_abs_rel_destination_path(dst_root, src): @@ -395,6 +413,9 @@ def generate_current_api_dir(api_dir, dst_dir): api_dir: the original api directory dst_dir: the api directory to be compared in temporary directory """ + contrib_dst = dst_dir.joinpath("contrib") + shutil.copytree(str(api_dir.joinpath("contrib")), str(contrib_dst)) + dst = dst_dir.joinpath("envoy") shutil.copytree(str(api_dir.joinpath("envoy")), str(dst)) diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index 8ac4ca1c43d60..0636bd08fbc75 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -23,6 +23,7 @@ py_binary( name = "protodoc", srcs = ["protodoc.py"], data = [ + "//contrib:extensions_metadata.yaml", "//docs:protodoc_manifest.yaml", "//docs:v2_mapping.json", "//source/extensions:extensions_metadata.yaml", diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index a95efe31b4b46..a8d45c7ccd1e5 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -56,7 +56,7 @@ .. _extension_{{extension}}: This extension may be referenced by the qualified name ``{{extension}}`` - +{{contrib}} .. note:: {{status}} @@ -77,12 +77,22 @@ .. _extension_category_{{category}}: .. tip:: +{% if extensions %} This extension category has the following known extensions: {% for ext in extensions %} - :ref:`{{ext}} ` {% endfor %} +{% endif %} +{% if contrib_extensions %} + The following extensions are available in :ref:`contrib ` images only: + +{% for ext in contrib_extensions %} + - :ref:`{{ext}} ` +{% endfor %} +{% endif %} + """) # A map from the extension security postures (as defined in the @@ -117,12 +127,20 @@ r = runfiles.Create() EXTENSION_DB = utils.from_yaml(r.Rlocation("envoy/source/extensions/extensions_metadata.yaml")) +CONTRIB_EXTENSION_DB = utils.from_yaml(r.Rlocation("envoy/contrib/extensions_metadata.yaml")) + # create an index of extension categories from extension db -EXTENSION_CATEGORIES = {} -for _k, _v in EXTENSION_DB.items(): - for _cat in _v['categories']: - EXTENSION_CATEGORIES.setdefault(_cat, []).append(_k) +def build_categories(extensions_db): + ret = {} + for _k, _v in extensions_db.items(): + for _cat in _v['categories']: + ret.setdefault(_cat, []).append(_k) + return ret + + +EXTENSION_CATEGORIES = build_categories(EXTENSION_DB) +CONTRIB_EXTENSION_CATEGORIES = build_categories(CONTRIB_EXTENSION_DB) V2_LINK_TEMPLATE = Template( """ @@ -241,18 +259,29 @@ def format_extension(extension): RST formatted extension description. """ try: - extension_metadata = EXTENSION_DB[extension] + extension_metadata = EXTENSION_DB.get(extension, None) + contrib = '' + if extension_metadata is None: + extension_metadata = CONTRIB_EXTENSION_DB[extension] + contrib = """ + +.. note:: + This extension is only available in :ref:`contrib ` images. + +""" status = EXTENSION_STATUS_VALUES.get(extension_metadata.get('status'), '') security_posture = EXTENSION_SECURITY_POSTURES[extension_metadata['security_posture']] categories = extension_metadata["categories"] except KeyError as e: sys.stderr.write( - f"\n\nDid you forget to add '{extension}' to source/extensions/extensions_build_config.bzl " - "or source/extensions/extensions_metadata.yaml?\n\n") + f"\n\nDid you forget to add '{extension}' to extensions_build_config.bzl, " + "extensions_metadata.yaml, contrib_build_config.bzl, " + "or contrib/extensions_metadata.yaml?\n\n") exit(1) # Raising the error buries the above message in tracebacks. return EXTENSION_TEMPLATE.render( extension=extension, + contrib=contrib, status=status, security_posture=security_posture, categories=categories) @@ -267,12 +296,14 @@ def format_extension_category(extension_category): Returns: RST formatted extension category description. """ - try: - extensions = EXTENSION_CATEGORIES[extension_category] - except KeyError as e: + extensions = EXTENSION_CATEGORIES.get(extension_category, []) + contrib_extensions = CONTRIB_EXTENSION_CATEGORIES.get(extension_category, []) + if not extensions and not contrib_extensions: raise ProtodocError(f"\n\nUnable to find extension category: {extension_category}\n\n") return EXTENSION_CATEGORY_TEMPLATE.render( - category=extension_category, extensions=sorted(extensions)) + category=extension_category, + extensions=sorted(extensions), + contrib_extensions=sorted(contrib_extensions)) def format_header_from_file(style, source_code_info, proto_name, v2_link): diff --git a/tools/type_whisperer/proto_build_targets_gen.py b/tools/type_whisperer/proto_build_targets_gen.py index 566febc443f33..2fd33ac0361c3 100644 --- a/tools/type_whisperer/proto_build_targets_gen.py +++ b/tools/type_whisperer/proto_build_targets_gen.py @@ -120,6 +120,11 @@ def is_v3_package(pkg): for desc in type_db.types.values(): pkg = desc.qualified_package if is_v3_package(pkg): + # contrib API files have the standard namespace but are in a contrib folder for clarity. + # The following prepends contrib to the package path which indirectly will produce the + # proper bazel path. + if desc.proto_path.startswith('contrib/'): + pkg = "contrib." + pkg v3_packages.add(pkg) continue if is_v2_package(pkg):