sync from master.

.azure-pipelines/pipelines.yml

@@ -18,6 +18,12 @@ stages:
     pool:
       vmImage: "ubuntu-18.04"
     steps:
+    - task: Cache@2
+      inputs:
+        key: "format_pre | ./WORKSPACE | **/*.bzl"
+        path: $(Build.StagingDirectory)/repository_cache
+      continueOnError: true
+
     - script: ci/run_envoy_docker.sh 'ci/do_ci.sh format_pre'
       workingDirectory: $(Build.SourcesDirectory)
       env:
@@ -38,6 +44,12 @@ stages:
     pool:
       vmImage: "ubuntu-18.04"
     steps:
+    - task: Cache@2
+      inputs:
+        key: "tooling | ./WORKSPACE | **/*.bzl"
+        path: $(Build.StagingDirectory)/repository_cache
+      continueOnError: true
+
     - script: ci/run_envoy_docker.sh 'ci/do_ci.sh tooling'
       workingDirectory: $(Build.SourcesDirectory)
       env:

.github/actions/pr_notifier/requirements.txt

@@ -115,9 +115,9 @@ slack-sdk==3.7.0 \
     --hash=sha256:50b9fd6d8f83af7e8ad6d8e76882d04931842241f85ccfd30da09b4a7b9b1516 \
     --hash=sha256:f0bf3e38ac393eba7fe1a99191b0e72f710860c6d2edc1271606fcfc08bea2e1
     # via -r .github/actions/pr_notifier/requirements.txt
-urllib3==1.26.5 \
-    --hash=sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c \
-    --hash=sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098
+urllib3==1.26.6 \
+    --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \
+    --hash=sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f
     # via requests
 wrapt==1.12.1 \
     --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7

CODEOWNERS

@@ -198,3 +198,5 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp
 /*/extensions/formatter/req_without_query @dio @tsaarni
 # IP address input matcher
 /*/extensions/matching/input_matchers/ip @aguinet @snowp
+# Kafka
+/*/extensions/filters/network/kafka @mattklein123 @adamkotwasinski

api/envoy/config/core/v3/protocol.proto

@@ -478,3 +478,11 @@ message Http3ProtocolOptions {
   // <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.stream_error_on_invalid_http_message>`.
   google.protobuf.BoolValue override_stream_error_on_invalid_http_message = 2;
 }
+
+// A message to control transformations to the :scheme header
+message SchemeHeaderTransformation {
+  oneof transformation {
+    // Overwrite any Scheme header with the contents of this string.
+    string scheme_to_overwrite = 1 [(validate.rules).string = {in: "http" in: "https"}];
+  }
+}

api/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto

@@ -30,7 +30,7 @@ message DnsCacheCircuitBreakers {
 
 // Configuration for the dynamic forward proxy DNS cache. See the :ref:`architecture overview
 // <arch_overview_http_dynamic_forward_proxy>` for more information.
-// [#next-free-field: 11]
+// [#next-free-field: 12]
 message DnsCacheConfig {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.common.dynamic_forward_proxy.v2alpha.DnsCacheConfig";
@@ -114,4 +114,10 @@ message DnsCacheConfig {
   // performance improvement, in the form of cache hits, for hostnames that are going to be
   // resolved during steady state and are known at config load time.
   repeated config.core.v3.SocketAddress preresolve_hostnames = 10;
+
+  // The timeout used for DNS queries. This timeout is independent of any timeout and retry policy
+  // used by the underlying DNS implementation (e.g., c-areas and Apple DNS) which are opaque.
+  // Setting this timeout will ensure that queries succeed or fail within the specified time frame
+  // and are then retried using the standard refresh rates. Defaults to 5s if not set.
+  google.protobuf.Duration dns_query_timeout = 11 [(validate.rules).duration = {gt {}}];
 }

api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto

@@ -35,7 +35,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // HTTP connection manager :ref:`configuration overview <config_http_conn_man>`.
 // [#extension: envoy.filters.network.http_connection_manager]
 
-// [#next-free-field: 48]
+// [#next-free-field: 49]
 message HttpConnectionManager {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager";
@@ -371,6 +371,11 @@ message HttpConnectionManager {
   ServerHeaderTransformation server_header_transformation = 34
       [(validate.rules).enum = {defined_only: true}];
 
+  // Allows for explicit transformation of the :scheme header on the request path.
+  // If not set, Envoy's default :ref:`scheme  <config_http_conn_man_headers_scheme>`
+  // handling applies.
+  config.core.v3.SchemeHeaderTransformation scheme_header_transformation = 48;
+
   // The maximum request headers size for incoming connections.
   // If unconfigured, the default max request headers allowed is 60 KiB.
   // Requests that exceed this limit will receive a 431 response.

api/envoy/extensions/request_id/uuid/v3/uuid.proto

@@ -40,4 +40,9 @@ message UuidRequestIdConfig {
   // stable sampling of traces, access logs, etc. will no longer work and only random sampling will
   // be possible.
   google.protobuf.BoolValue pack_trace_reason = 1;
+
+  // Set whether to use :ref:`x-request-id<config_http_conn_man_headers_x-request-id>` for sampling or not.
+  // This defaults to true. See the :ref:`context propagation <arch_overview_tracing_context_propagation>`
+  // overview for more information.
+  google.protobuf.BoolValue use_request_id_for_trace_sampling = 2;
 }

bazel/README.md

@@ -863,7 +863,7 @@ TEST_TMPDIR=/tmp tools/gen_compilation_database.py
 ```
 
 
-# Running clang-format without docker
+# Running format linting without docker
 
 The easiest way to run the clang-format check/fix commands is to run them via
 docker, which helps ensure the right toolchain is set up. However you may prefer
@@ -876,6 +876,8 @@ To run the tools directly, you must install the correct version of clang. This
 may change over time, check the version of clang in the docker image. You must
 also have 'buildifier' installed from the bazel distribution.
 
+Note that if you run the `check_spelling.py` script you will need to have `aspell` installed.
+
 Edit the paths shown here to reflect the installation locations on your system:
 
 ```shell

bazel/foreign_cc/BUILD

@@ -315,9 +315,9 @@ envoy_cmake_external(
     name = "wamr",
     cache_entries = {
         "LLVM_DIR": "$EXT_BUILD_DEPS/copy_llvm/llvm/lib/cmake/llvm",
-        "WAMR_BUILD_INTERP": "1",
-        "WAMR_BUILD_JIT": "0",
-        "WAMR_BUILD_AOT": "0",
+        "WAMR_BUILD_INTERP": "0",
+        "WAMR_BUILD_JIT": "1",
+        "WAMR_BUILD_AOT": "1",
         "WAMR_BUILD_SIMD": "0",
         "WAMR_BUILD_MULTI_MODULE": "1",
         "WAMR_BUILD_LIBC_WASI": "0",

bazel/repository_locations.bzl

@@ -708,11 +708,11 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_name = "Webassembly Micro Runtime",
         project_desc = "A standalone runtime with a small footprint for WebAssembly",
         project_url = "https://github.com/bytecodealliance/wasm-micro-runtime",
-        version = "a14a4487bb8b493bf6c68d83b03f12028d16f58a",
-        sha256 = "d68668e129f16a9ddd7a1a0da22b17905a25001ae2de398726d37880b61fee9e",
+        version = "b554a9d05d89bb4ef28068b4ae4d0ee6c99bc9db",
+        sha256 = "de6b68118c5d4b0d37c9049fa08fae6a850304522ec307f087f0eca4ad8fff57",
         strip_prefix = "wasm-micro-runtime-{version}",
         urls = ["https://github.com/bytecodealliance/wasm-micro-runtime/archive/{version}.tar.gz"],
-        release_date = "2021-05-14",
+        release_date = "2021-07-06",
         use_category = ["dataplane_ext"],
         extensions = ["envoy.wasm.runtime.wamr"],
         cpe = "N/A",

configs/envoyproxy_io_proxy.yaml

@@ -17,6 +17,8 @@ static_resources:
       - name: envoy.filters.network.http_connection_manager
         typed_config:
           "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          scheme_header_transformation:
+            scheme_to_overwrite: https
           stat_prefix: ingress_http
           route_config:
             name: local_route

docs/root/configuration/http/http_conn_man/headers.rst

@@ -9,6 +9,25 @@ is being received) as well as during encoding (when the response is being sent).
 .. contents::
   :local:
 
+.. _config_http_conn_man_headers_scheme:
+
+:scheme
+-------
+
+Envoy will always set the *:scheme* header while processing a request. It should always be available to filters, and should be forwarded upstream for HTTP/2 and HTTP/3, where :ref:`config_http_conn_man_headers_x-forwarded-proto` will be sent for HTTP/1.1.
+
+For HTTP/2, and HTTP/3, incoming *:scheme* headers are trusted and propogated through upstream.
+For HTTP/1, the *:scheme* header will be set
+1) From the absolute URL if present and valid. An invalid (not "http" or "https") scheme, or an https scheme over an unencrypted connection will result in Envoy rejecting the request. This is the only scheme validation Envoy performs as it avoids a HTTP/1.1-specific privledge escalation attack for edge Envoys [1]_ which doesn't have a comparable vector for HTTP/2 and above [2]_.
+2) From the value of the :ref:`config_http_conn_man_headers_x-forwarded-proto` header after sanitization (to valid *x-forwarded-proto* from trusted downstreams, otherwise based on downstream encryption level).
+
+This default behavior can be overridden via the :ref:`scheme_header_transformation
+<envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.scheme_header_transformation>`
+configuration option.
+
+.. [1] Edge Envoys often have plaintext HTTP/1.1 listeners. If Envoy trusts absolute URL scheme from fully qualfied URLs, a MiTM can adjust relative URLs to https absolute URLs, and inadvertantly cause the Envoy's upstream to send PII or other sensitive data over what it then believes is a secure connection.
+.. [2] Unlike HTTP/1.1, HTTP/2 is in practice always served over TLS via ALPN for edge Envoys. In mesh networks using insecure HTTP/2, if the downstream is not trusted to set scheme, the :ref:`scheme_header_transformation <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.scheme_header_transformation>` should be used.
+
 .. _config_http_conn_man_headers_user-agent:
 
 user-agent
@@ -341,6 +360,14 @@ It is a common case where a service wants to know what the originating protocol
 of the connection terminated by front/edge Envoy. *x-forwarded-proto* contains this information. It
 will be set to either *http* or *https*.
 
+Downstream *x-forwarded-proto* headers will only be trusted if *xff_num_trusted_hops* is non-zero.
+If *xff_num_trusted_hops* is zero, downstream *x-forwarded-proto* headers and *:scheme* headers
+will be set to http or https based on if the downstream connection is TLS or not.
+
+If the scheme is changed via the :ref:`scheme_header_transformation
+<envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.scheme_header_transformation>`
+configuration option, *x-forwarded-proto* will be updated as well.
+
 .. _config_http_conn_man_headers_x-request-id:
 
 x-request-id

docs/root/configuration/http/http_filters/dynamic_forward_proxy_filter.rst

@@ -49,6 +49,7 @@ namespace.
   dns_query_attempt, Counter, Number of DNS query attempts.
   dns_query_success, Counter, Number of DNS query successes.
   dns_query_failure, Counter, Number of DNS query failures.
+  dns_query_timeout, Counter, Number of DNS query :ref:`timeouts <envoy_v3_api_field_extensions.common.dynamic_forward_proxy.v3.DnsCacheConfig.dns_query_timeout>`.
   host_address_changed, Counter, Number of DNS queries that resulted in a host address change.
   host_added, Counter, Number of hosts that have been added to the cache.
   host_removed, Counter, Number of hosts that have been removed from the cache.

docs/root/configuration/http/http_filters/rate_limit_filter.rst

@@ -88,9 +88,9 @@ will be appended to the descriptor produced by the action and sent to the rateli
 overriding the static service configuration.
 
 The override can be configured to be taken from the :ref:`Dynamic Metadata
-<envoy_v3_api_msg_config.core.v3.Metadata>` under a specified :ref: `key
-<envoy_v3_api_msg_config.type.metadata.v3.MetadataKey>`. If the value is misconfigured
-or key does not exist, the override configuration is ignored.
+<envoy_v3_api_msg_config.core.v3.Metadata>` under a specified
+:ref:`key <envoy_v3_api_msg_type.metadata.v3.MetadataKey>`.
+If the value is misconfigured or key does not exist, the override configuration is ignored.
 
 Example 3
 ^^^^^^^^^

docs/root/intro/arch_overview/advanced/attributes.rst

@@ -40,7 +40,9 @@ Request attributes
 ------------------
 
 The following request attributes are generally available upon initial request
-processing, which makes them suitable for RBAC policies:
+processing, which makes them suitable for RBAC policies.
+
+``request.*`` attributes are only available in http filters.
 
 .. csv-table::
    :header: Attribute, Type, Description
@@ -76,6 +78,8 @@ Response attributes
 
 Response attributes are only available after the request completes.
 
+``response.*`` attributes are only available in http filters.
+
 .. csv-table::
    :header: Attribute, Type, Description
    :widths: 1, 1, 4

docs/root/intro/arch_overview/http/upgrades.rst

@@ -96,7 +96,7 @@ will synthesize 200 response headers, and then forward the TCP data as the HTTP
   will be forwarded *unsanitized* headers if they are in the body payload. Please use with caution
 
 For an example of proxying connect, please see :repo:`configs/proxy_connect.yaml <configs/proxy_connect.yaml>`
-For an example of terminating connect, please see :repo:`configs/terminate_connect.yaml <configs/terminate_connect.yaml>`
+For an example of terminating connect, please see :repo:`configs/terminate_http1_connect.yaml <configs/terminate_http1_connect.yaml>` and :repo:`configs/terminate_http2_connect.yaml <configs/terminate_http2_connect.yaml>`
 
 Note that for CONNECT-over-tls, Envoy can not currently be configured to do the CONNECT request in the clear
 and encrypt previously unencrypted payload in one hop. To send CONNECT in plaintext and encrypt the payload,

docs/root/intro/arch_overview/listeners/network_filter_chain.rst

@@ -16,7 +16,7 @@ chosen to serve the request. If the default filter chain is not supplied, the co
 Filter chain only update
 ------------------------
 
-:ref:`Filter chains <envoy_v3_api_msg_config.listener.v3.FilterChain>` can be updated indepedently. Upon listener config
+:ref:`Filter chains <envoy_v3_api_msg_config.listener.v3.FilterChain>` can be updated independently. Upon listener config
 update, if the listener manager determines that the listener update is a filter chain only update, the listener update
 will be executed by adding, updating and removing filter chains. The connections owned by these destroying filter chains will
 be drained as described in listener drain.

docs/root/intro/arch_overview/observability/tracing.rst

@@ -69,6 +69,16 @@ to be correlated.
   field can be used to disable this behavior at the expense of also disabling stable trace reason
   propagation and associated features within a deployment.
 
+.. attention::
+
+  The sampling policy for Envoy is determined by the value of :ref:`x-request-id <config_http_conn_man_headers_x-request-id>` by default.
+  However, such a sampling policy is only valid for a fleet of Envoys. If a service proxy
+  that is not Envoy is present in the fleet, sampling is performed without considering the policy of that proxy.
+  For meshes consisting of multiple service proxies such as this, it is more effective to
+  bypass Envoy's sampling policy and sample based on the trace provider's sampling policy. This can be achieved by setting
+  :ref:`use_request_id_for_trace_sampling <envoy_v3_api_field_extensions.request_id.uuid.v3.UuidRequestIdConfig.use_request_id_for_trace_sampling>`
+  to false.
+
 The tracing providers also require additional context, to enable the parent/child relationships
 between the spans (logical units of work) to be understood. This can be achieved by using the
 LightStep (via OpenTracing API) or Zipkin tracer directly within the service itself, to extract the