Add response trailers for bandwidth limit filter

.azure-pipelines/pipelines.yml

@@ -381,6 +381,15 @@ stages:
         GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
       displayName: "Generate docs"
 
+    - script: |
+        ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs'
+      displayName: "Upload Docs to GCS"
+      env:
+        ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+        GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+        GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
+      condition: eq(variables['Build.SourceBranch'], 'refs/heads/main')
+
     - task: InstallSSHKey@0
       inputs:
         hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
@@ -393,6 +402,7 @@ stages:
       workingDirectory: $(Build.SourcesDirectory)
       env:
         AZP_BRANCH: $(Build.SourceBranch)
+        NETLIFY_TRIGGER_URL: $(NetlifyTriggerURL)
 
 - stage: verify
   dependsOn: ["docker"]
@@ -454,7 +464,7 @@ stages:
         testRunTitle: "macOS"
       condition: always()
 
-    - script: ./ci/flaky_test/run_process_xml.sh
+    - script: bazel run //ci/flaky_test:process_xml
       displayName: "Process Test Results"
       env:
         TEST_TMPDIR: $(Build.SourcesDirectory)

.github/actions/pr_notifier/pr_notifier.py

@@ -69,6 +69,10 @@ def is_waiting(labels):
     return False
 
 
+def is_contrib(labels):
+    return any(label.name == "contrib" for label in labels)
+
+
 # Return true if the PR has an API tag, false otherwise.
 def is_api(labels):
     for label in labels:
@@ -174,7 +178,7 @@ def track_prs():
             pr_info.assignees, maintainers_and_prs, message, MAINTAINERS, FIRST_PASS)
 
         # If there was no maintainer, track it as unassigned.
-        if not has_maintainer_assignee:
+        if not has_maintainer_assignee and not is_contrib(labels):
             maintainers_and_prs['unassigned'] = maintainers_and_prs['unassigned'] + message
 
     # Return the dict of {maintainers : PR notifications},

.github/actions/pr_notifier/requirements.in

@@ -0,0 +1,2 @@
+pygithub
+slack_sdk

.github/actions/pr_notifier/requirements.txt

@@ -63,9 +63,9 @@ chardet==4.0.0 \
     --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \
     --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5
     # via requests
-deprecated==1.2.12 \
-    --hash=sha256:08452d69b6b5bc66e8330adde0a4f8642e969b9e1702904d137eeb29c8ffc771 \
-    --hash=sha256:6d2de2de7931a968874481ef30208fd4e08da39177d61d3d4ebdf4366e7dbca1
+deprecated==1.2.13 \
+    --hash=sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d \
+    --hash=sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d
     # via pygithub
 idna==2.10 \
     --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \
@@ -78,7 +78,7 @@ pycparser==2.20 \
 pygithub==1.55 \
     --hash=sha256:1bbfff9372047ff3f21d5cd8e07720f3dbfdaf6462fcaed9d815f528f1ba7283 \
     --hash=sha256:2caf0054ea079b71e539741ae56c5a95e073b81fa472ce222e81667381b9601b
-    # via -r .github/actions/pr_notifier/requirements.txt
+    # via -r requirements.in
 pyjwt==2.1.0 \
     --hash=sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1 \
     --hash=sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130
@@ -111,10 +111,10 @@ six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via pynacl
-slack-sdk==3.10.1 \
-    --hash=sha256:f17b71a578e94204d9033bffded634475f4ca0a6274c6c7a4fd8a9cb0ac7cd8b \
-    --hash=sha256:2b4dde7728eb4ff5a581025d204578ccff25a5d8f0fe11ae175e3ce6e074434f
-    # via -r .github/actions/pr_notifier/requirements.txt
+slack_sdk==3.11.0 \
+    --hash=sha256:4d9854ee158c3137cfe1ba587dc4b777b6881aee58436d8071f36bad842acbf4 \
+    --hash=sha256:8dc858cd106b639191ee3dc38fb957e55ab8dd28c2cc22feafa1223ab2def646
+    # via -r requirements.in
 urllib3==1.26.6 \
     --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \
     --hash=sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f

.github/dependabot.yml

@@ -7,12 +7,12 @@ updates:
     interval: "daily"
 
 - package-ecosystem: "pip"
-  directory: "/test/extensions/filters/network/thrift_proxy"
+  directory: "/configs"
   schedule:
     interval: "daily"
 
 - package-ecosystem: "pip"
-  directory: "/contrib/kafka/filters/network/source"
+  directory: "/test/extensions/filters/network/thrift_proxy"
   schedule:
     interval: "daily"
 
@@ -26,63 +26,13 @@ updates:
   schedule:
     interval: "daily"
 
-- package-ecosystem: "pip"
-  directory: "/tools/docs"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/git"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/config_validation"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/dependency"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/deprecate_version"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/distribution"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/protodoc"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/deprecate_features"
-  schedule:
-    interval: "daily"
-
 - package-ecosystem: "pip"
   directory: "/tools/code_format"
   schedule:
     interval: "daily"
 
 - package-ecosystem: "pip"
-  directory: "/ci/flaky_test"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/configs"
-  schedule:
-    interval: "daily"
-
-- package-ecosystem: "pip"
-  directory: "/tools/testing"
+  directory: "/tools/dependency"
   schedule:
     interval: "daily"
 

BUILD

@@ -8,6 +8,11 @@ exports_files([
     ".coveragerc",
 ])
 
+alias(
+    name = "envoy",
+    actual = "//source/exe:envoy",
+)
+
 # These two definitions exist to help reduce Envoy upstream core code depending on extensions.
 # To avoid visibility problems, see notes in source/extensions/extensions_build_config.bzl
 #

CONTRIBUTING.md

@@ -253,15 +253,9 @@ Extension configuration should be located in a directory structure like
 The code for the extension should be located under the equivalent
 `source/extensions/area/plugin`, and include an *envoy_cc_extension* with the
 configuration and tagged with the appropriate security posture, and an
-*envoy_cc_library* with the code. More details on how to add a new extension
-API can be found [here](api/STYLE.md#adding-an-extension-configuration-to-the-api):
+*envoy_cc_library* with the code.
 
-Other changes will likely include
-
-  * Editing [source/extensions/extensions_build_config.bzl](source/extensions/extensions_build_config.bzl) to include the new extensions
-  * Editing [source/extensions/extensions_metadata.yaml](source/extensions/extensions_metadata.yaml) to include metadata for the new extensions
-  * Editing [docs/root/api-v3/config/config.rst](docs/root/api-v3/config/config.rst) to add area/area
-  * Adding `docs/root/api-v3/config/area/area.rst` to add a table of contents for the API docs
+More details on how to add a new extension API can be found [here](api/STYLE.md#adding-an-extension-configuration-to-the-api):
 
 # Adding contrib extensions
 

GOVERNANCE.md

@@ -140,7 +140,7 @@ New Features
 Deprecated
 ----------
 ```
-* Run the deprecate_versions.py script (e.g. `sh tools/deprecate_version/deprecate_version.sh`)
+* Run the deprecate_versions.py script (e.g. `bazel run //tools/deprecate_version:deprecate_version`)
   to file tracking issues for runtime guarded code which can be removed.
 * Check source/common/runtime/runtime_features.cc and see if any runtime guards in
   disabled_runtime_features should be reassessed, and ping on the relevant issues.

OWNERS.md

@@ -65,6 +65,7 @@ without further review.
 * Tony Allen ([tonya11en](https://github.com/tonya11en)) (tony@allen.gg)
 * Dmitri Dolguikh ([dmitri-d](https://github.com/dmitri-d)) (ddolguik@redhat.com)
 * Yan Avlasov ([yanavlasov](https://github.com/yanavlasov)) (yavlasov@google.com)
+* William A Rowe Jr ([wrowe](https://github.com/wrowe)) (wrowe@vmware.com)
 
 # Emeritus maintainers
 

WORKSPACE

@@ -16,6 +16,10 @@ load("//bazel:repositories_extra.bzl", "envoy_dependencies_extra")
 
 envoy_dependencies_extra()
 
+load("@base_pip3//:requirements.bzl", "install_deps")
+
+install_deps()
+
 load("//bazel:dependency_imports.bzl", "envoy_dependency_imports")
 
 envoy_dependency_imports()

api/BUILD

@@ -60,13 +60,15 @@ proto_library(
         "//contrib/envoy/extensions/filters/http/squash/v3:pkg",
         "//contrib/envoy/extensions/filters/http/sxg/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/network/kafka_broker/v3:pkg",
+        "//contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/network/mysql_proxy/v3:pkg",
         "//contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha:pkg",
         "//contrib/envoy/extensions/filters/network/rocketmq_proxy/v3:pkg",
         "//envoy/admin/v3:pkg",
         "//envoy/config/accesslog/v3:pkg",
         "//envoy/config/bootstrap/v3:pkg",
         "//envoy/config/cluster/v3:pkg",
+        "//envoy/config/common/key_value/v3:pkg",
         "//envoy/config/common/matcher/v3:pkg",
         "//envoy/config/core/v3:pkg",
         "//envoy/config/endpoint/v3:pkg",
@@ -100,7 +102,6 @@ proto_library(
         "//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg",
         "//envoy/extensions/clusters/redis/v3:pkg",
         "//envoy/extensions/common/dynamic_forward_proxy/v3:pkg",
-        "//envoy/extensions/common/key_value/v3:pkg",
         "//envoy/extensions/common/matching/v3:pkg",
         "//envoy/extensions/common/ratelimit/v3:pkg",
         "//envoy/extensions/common/tap/v3:pkg",

api/STYLE.md

@@ -112,10 +112,9 @@ Extensions must currently be added as v3 APIs following the [package
 organization](#package-organization) above.
 To add an extension config to the API, the steps below should be followed:
 
-1. If this is still WiP and subject to breaking changes, use `vNalpha` instead of `vN` in steps
-   below. Refer to the [Cache filter config](envoy/extensions/filters/http/cache/v3alpha/cache.proto)
-   as an example of `v3alpha`, and the
-   [Buffer filter config](envoy/extensions/filters/http/buffer/v3/buffer.proto) as an example of `v3`.
+1. If this is still WiP and subject to breaking changes, please tag it
+   `option (udpa.annotations.file_status).work_in_progress = true;` and
+   optionally hide it from the docs (`[#not-implemented-hide:]`.
 1. Place the v3 extension configuration `.proto` in `api/envoy/extensions`, e.g.
    `api/envoy/extensions/filters/http/foobar/v3/foobar.proto` together with an initial BUILD file:
    ```bazel
@@ -127,17 +126,26 @@ To add an extension config to the API, the steps below should be followed:
        deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
    )
    ```
-1. Add to the v3 extension config proto `import "udpa/annotations/status.proto";`
-1. If this is still WiP and subject to breaking changes, set
-   `option (udpa.annotations.file_status).work_in_progress = true;`.
-1. Add to the v3 extension config proto a file level
-   `option (udpa.annotations.file_status).package_version_status = ACTIVE;`.
+1. Update [source/extensions/extensions_metadata.yaml](../source/extensions/extensions_metadata.yaml)
+   with the category, security posture, and status. The category field will have to match an
+   annotation of the form `// [#extension-category: your.extension.category]`
+   in one of the proto files for the docs build to pass.
+1. Update
+   [source/extensions/extensions_build_config.bzl](source/extensions/extensions_build_config.bzl)
+   to include the new extension.
+1. If the extension is not hidden, find or create a docs file with a toctree
+   and to reference your proto to make sure users can navigate to it from the API docs
+   (and to not break the docs build).
+   See the [key-value-store PR](https://github.com/envoyproxy/envoy/pull/17745/files) for an example of adding a new extension point to common.
+1. Make sure your proto imports the v3 extension config proto (`import "udpa/annotations/status.proto";`)
+1. Make sure your proto is either tracked as a work in progress
+   (`option (udpa.annotations.file_status).work_in_progress = true;`)
+   or ready to be used
+   (`option (udpa.annotations.file_status).package_version_status = ACTIVE;`).
    This is required to automatically include the config proto in [api/versioning/BUILD](versioning/BUILD).
 1. Add a reference to the v3 extension config in (1) in [api/versioning/BUILD](versioning/BUILD) under `active_protos`.
-1. Run `./tools/proto_format/proto_format.sh fix`. This should regenerate the `BUILD` file,
-   reformat `foobar.proto` as needed and also generate the v4alpha extension config (if needed),
-   together with shadow API protos.
-1. `git add api/ generated_api_shadow/` to add any new files to your Git index.
+1. Run `./tools/proto_format/proto_format.sh fix`. This should regenerate the `BUILD` file and
+   reformat `foobar.proto` as needed.
 
 ## API annotations
 

api/bazel/BUILD

@@ -1,4 +1,6 @@
 load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler")
+load("//:utils.bzl", "json_data")
+load(":repository_locations.bzl", "REPOSITORY_LOCATIONS_SPEC")
 
 licenses(["notice"])  # Apache 2
 
@@ -15,3 +17,8 @@ go_proto_compiler(
     valid_archive = False,
     visibility = ["//visibility:public"],
 )
+
+json_data(
+    name = "repository_locations",
+    data = REPOSITORY_LOCATIONS_SPEC,
+)

api/contrib/envoy/extensions/filters/network/kafka_mesh/v3alpha/kafka_mesh.proto

@@ -0,0 +1,58 @@
+syntax = "proto3";
+
+package envoy.extensions.filters.network.kafka_mesh.v3alpha;
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.filters.network.kafka_mesh.v3alpha";
+option java_outer_classname = "KafkaMeshProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).work_in_progress = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Kafka Mesh]
+// Kafka Mesh :ref:`configuration overview <config_network_filters_kafka_mesh>`.
+// [#extension: envoy.filters.network.kafka_mesh]
+
+message KafkaMesh {
+  // Envoy's host that's advertised to clients.
+  // Has the same meaning as corresponding Kafka broker properties.
+  // Usually equal to filter chain's listener config, but needs to be reachable by clients
+  // (so 0.0.0.0 will not work).
+  string advertised_host = 1 [(validate.rules).string = {min_len: 1}];
+
+  // Envoy's port that's advertised to clients.
+  int32 advertised_port = 2 [(validate.rules).int32 = {gt: 0}];
+
+  // Upstream clusters this filter will connect to.
+  repeated KafkaClusterDefinition upstream_clusters = 3;
+
+  // Rules that will decide which cluster gets which request.
+  repeated ForwardingRule forwarding_rules = 4;
+}
+
+message KafkaClusterDefinition {
+  // Cluster name.
+  string cluster_name = 1 [(validate.rules).string = {min_len: 1}];
+
+  // Kafka cluster address.
+  string bootstrap_servers = 2 [(validate.rules).string = {min_len: 1}];
+
+  // Default number of partitions present in this cluster.
+  // This is especially important for clients that do not specify partition in their payloads and depend on this value for hashing.
+  int32 partition_count = 3 [(validate.rules).int32 = {gt: 0}];
+
+  // Custom configuration passed to Kafka producer.
+  map<string, string> producer_config = 4;
+}
+
+message ForwardingRule {
+  // Cluster name.
+  string target_cluster = 1;
+
+  oneof trigger {
+    // Intended place for future types of forwarding rules.
+    string topic_prefix = 2;
+  }
+}