sync from master

.azure-pipelines/cve_scan.yml

@@ -4,14 +4,13 @@
 trigger: none
 pr: none
 
-# This appears to be broken right now so disabling until it is fixed.
-# schedules:
-# - cron: "0 * * * *"
-#   displayName: Hourly CVE scan
-#   branches:
-#     include:
-#     - main
-#   always: true
+schedules:
+- cron: "0 * * * *"
+  displayName: Hourly CVE scan
+  branches:
+    include:
+    - main
+  always: true
 
 pool:
   vmImage: "ubuntu-18.04"

api/buf.yaml

@@ -1,6 +1,6 @@
 version: v1beta1
 deps:
-    - buf.build/googleapis/googleapis
+    - buf.build/googleapis/googleapis:d1a849b8f8304950832335723096e954
     - buf.build/beta/opencensus
     - buf.build/beta/prometheus
     - buf.build/beta/opentelemetry

api/envoy/extensions/filters/http/ext_proc/v3alpha/ext_proc.proto

@@ -150,7 +150,6 @@ message ExternalProcessor {
   string stat_prefix = 8;
 }
 
-// [#not-implemented-hide:]
 // Extra settings that may be added to per-route configuration for a
 // virtual host or cluster.
 message ExtProcPerRoute {
@@ -161,23 +160,27 @@ message ExtProcPerRoute {
     // If disabled is specified in multiple per-filter-configs, the most specific one will be used.
     bool disabled = 1 [(validate.rules).bool = {const: true}];
 
-    // Override aspects of the configuration for this route
+    // Override aspects of the configuration for this route. A set of
+    // overrides in a more specific configuration will override a "disabled"
+    // flag set in a less-specific one.
     ExtProcOverrides overrides = 2;
   }
 }
 
-// [#not-implemented-hide:]
 // Overrides that may be set on a per-route basis
 message ExtProcOverrides {
   // Set a different processing mode for this route than the default.
   ProcessingMode processing_mode = 1;
 
+  // [#not-implemented-hide:]
   // Set a different asynchronous processing option than the default.
   bool async_mode = 2;
 
+  // [#not-implemented-hide:]
   // Set different optional properties than the default.
   repeated string request_properties = 3;
 
+  // [#not-implemented-hide:]
   // Set different optional properties than the default.
   repeated string response_properties = 4;
 }

api/envoy/extensions/filters/udp/dns_filter/v3alpha/BUILD

@@ -6,6 +6,7 @@ licenses(["notice"])  # Apache 2
 
 api_proto_package(
     deps = [
+        "//envoy/annotations:pkg",
         "//envoy/config/core/v3:pkg",
         "//envoy/data/dns/v3:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",

api/envoy/extensions/filters/udp/dns_filter/v3alpha/dns_filter.proto

@@ -2,12 +2,14 @@ syntax = "proto3";
 
 package envoy.extensions.filters.udp.dns_filter.v3alpha;
 
+import "envoy/config/core/v3/address.proto";
 import "envoy/config/core/v3/base.proto";
 import "envoy/config/core/v3/resolver.proto";
 import "envoy/data/dns/v3/dns_table.proto";
 
 import "google/protobuf/duration.proto";
 
+import "envoy/annotations/deprecation.proto";
 import "udpa/annotations/status.proto";
 import "validate/validate.proto";
 
@@ -44,15 +46,29 @@ message DnsFilterConfig {
   // in a client context. This message will contain the timeouts, retry,
   // and forwarding configuration for Envoy to make DNS requests to other
   // resolvers
+  //
+  // [#next-free-field: 6]
   message ClientContextConfig {
     // Sets the maximum time we will wait for the upstream query to complete
     // We allow 5s for the upstream resolution to complete, so the minimum
     // value here is 1. Note that the total latency for a failed query is the
     // number of retries multiplied by the resolver_timeout.
     google.protobuf.Duration resolver_timeout = 1 [(validate.rules).duration = {gte {seconds: 1}}];
 
+    // This field was used for `dns_resolution_config` in Envoy 1.19.0 and
+    // 1.19.1.
+    // Control planes that need to set this field for Envoy 1.19.0 and
+    // 1.19.1 clients should fork the protobufs and change the field type
+    // to `DnsResolutionConfig`.
+    // Control planes that need to simultaneously support Envoy 1.18.x and
+    // Envoy 1.19.x should avoid Envoy 1.19.0 and 1.19.1.
+    //
+    // [#not-implemented-hide:]
+    repeated config.core.v3.Address upstream_resolvers = 2
+        [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"];
+
     // DNS resolution configuration which includes the underlying dns resolver addresses and options.
-    config.core.v3.DnsResolutionConfig dns_resolution_config = 2;
+    config.core.v3.DnsResolutionConfig dns_resolution_config = 5;
 
     // Controls how many outstanding external lookup contexts the filter tracks.
     // The context structure allows the filter to respond to every query even if the external

bazel/README.md

@@ -7,7 +7,7 @@ It is recommended to use [Bazelisk](https://github.com/bazelbuild/bazelisk) inst
 On Linux, run the following commands:
 
 ```console
-sudo wget -O /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64
+sudo wget -O /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-$([ $(uname -m) = "aarch64" ] && echo "arm64" || echo "amd64")
 sudo chmod +x /usr/local/bin/bazel
 ```
 

docs/root/api-docs/xds_protocol.rst

@@ -851,13 +851,32 @@ names, which the server thought the client was already not subscribed
 to. The server must cleanly process such a request; it can simply ignore
 these phantom unsubscriptions.
 
+In most cases (see below for exception), a server does not need to send any response if a request
+does nothing except unsubscribe from a resource; in particular, servers are not generally required
+to send a response with the unsubscribed resource name in the
+:ref:`removed_resources <envoy_v3_api_field_service.discovery.v3.DeltaDiscoveryResponse.removed_resources>`
+field.
+
+However, there is one exception to the above: When a client has a wildcard subscription ("*") *and*
+a subscription to another specific resource name, it is possible that the specific resource name is
+also included in the wildcard subscription, so if the client unsubscribes from that specific
+resource name, it does not know whether or not to continue to cache the resource. To address this,
+the server must send a response that includes the specific resource in either the
+:ref:`removed_resources
+<envoy_v3_api_field_service.discovery.v3.DeltaDiscoveryResponse.removed_resources>`
+field (if it is not included in the wildcard) or in the
+:ref:`resources <envoy_v3_api_field_service.discovery.v3.DeltaDiscoveryResponse.resources>`
+field (if it *is* included in the wildcard).
+
 Knowing When a Requested Resource Does Not Exist
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-When a resource subscribed to by a client does not exist, the server will send a :ref:`Resource
-<envoy_v3_api_msg_service.discovery.v3.Resource>` whose :ref:`name <envoy_v3_api_field_service.discovery.v3.Resource.name>` field matches the
-name that the client subscribed to and whose :ref:`resource <envoy_v3_api_msg_service.discovery.v3.Resource>`
-field is unset. This allows the client to quickly determine when a resource does not exist without
+When a resource subscribed to by a client does not exist, the server
+will send a
+:ref:`DeltaDiscoveryResponse <envoy_v3_api_msg_service.discovery.v3.DeltaDiscoveryResponse>`
+message that contains that resource's name in the
+:ref:`removed_resources <envoy_v3_api_field_service.discovery.v3.DeltaDiscoveryResponse.removed_resources>`
+field. This allows the client to quickly determine when a resource does not exist without
 waiting for a timeout, as would be done in the SotW protocol variants. However, clients are still
 encouraged to use a timeout to protect against the case where the management server fails to send
 a response in a timely manner.

docs/root/configuration/operations/overload_manager/overload_manager.rst

@@ -234,12 +234,19 @@ streams based on heap usage as a trigger. When the heap usage is less than 85%,
 no streams will be reset.  When heap usage is at or above 85%, we start to
 reset buckets according to the strategy described below. When the heap
 usage is at 95% all streams using >= 1MiB memory are eligible for reset.
+This overload action will reset up to 50 streams (this is a hardcoded limit)
+per worker everytime the action is invoked. This is both to reduce the amount
+of streams that end up getting reset and to prevent the worker thread from
+locking up and triggering the Watchdog system.
 
 Given that there are only 8 buckets, we partition the space with a gradation of
 :math:`gradation = (saturation_threshold - scaling_threshold)/8`. Hence at 85%
 heap usage we reset streams in the last bucket e.g. those using `>= 128MiB`. At
 :math:`85% + 1 * gradation` heap usage we reset streams in the last two buckets
-e.g. those using `>= 64MiB`. And so forth as the heap usage is higher.
+e.g. those using `>= 64MiB`, prioritizing the streams in the last bucket since
+there's a hard limit on the number of streams we can reset per invokation.
+At :math:`85% + 2 * gradation` heap usage we reset streams in the last three
+buckets e.g. those using `>= 32MiB`. And so forth as the heap usage is higher.
 
 It's expected that the first few gradations shouldn't trigger anything, unless
 there's something seriously wrong e.g. in this example streams using `>=

docs/root/version_history/current.rst

@@ -17,6 +17,15 @@ Incompatible Behavior Changes
   :ref:`contrib images <install_contrib>`.
 * contrib: the :ref:`MySQL proxy filter <config_network_filters_mysql_proxy>` has been moved to
   :ref:`contrib images <install_contrib>`.
+* dns_filter: :ref:`dns_filter <envoy_v3_api_msg_extensions.filters.udp.dns_filter.v3alpha.DnsFilterConfig>`
+  protobuf fields have been renumbered to restore compatibility with Envoy
+  1.18, breaking compatibility with Envoy 1.19.0 and 1.19.1. The new field
+  numbering allows control planes supporting Envoy 1.18 to gracefully upgrade to
+  :ref:`dns_resolution_config <envoy_v3_api_field_extensions.filters.udp.dns_filter.v3alpha.DnsFilterConfig.ClientContextConfig.dns_resolution_config>`,
+  provided they skip over Envoy 1.19.0 and 1.19.1.
+  Control planes upgrading from Envoy 1.19.0 and 1.19.1 will need to
+  vendor the corresponding protobuf definitions to ensure that the
+  renumbered fields have the types expected by those releases.
 * ext_authz: fixed skipping authentication when returning either a direct response or a redirect. This behavior can be temporarily reverted by setting the ``envoy.reloadable_features.http_ext_authz_do_not_skip_direct_response_and_redirect`` runtime guard to false.
 
 Minor Behavior Changes

envoy/common/conn_pool.h

@@ -36,6 +36,21 @@ class Cancellable {
   virtual void cancel(CancelPolicy cancel_policy) PURE;
 };
 
+/**
+ * Controls the behavior when draining a connection pool.
+ */
+enum class DrainBehavior {
+  // Starts draining a pool, by gracefully completing all requests and gracefully closing all
+  // connections, in preparation for deletion. It is invalid to create new streams or
+  // connections from this pool after draining a pool with this behavior.
+  DrainAndDelete,
+  // Actively drain all existing connection pool connections. This can be used in cases where
+  // the connection pool is not being destroyed, but the caller wishes to make sure that
+  // all new streams take place on a new connection. For example, when a health check failure
+  // occurs.
+  DrainExistingConnections,
+};
+
 /**
  * An instance of a generic connection pool.
  */
@@ -59,20 +74,10 @@ class Instance {
   virtual bool isIdle() const PURE;
 
   /**
-   * Starts draining a pool, by gracefully completing all requests and gracefully closing all
-   * connections, in preparation for deletion. When the process completes, the function registered
-   * via `addIdleCallback()` is called. The callback may occur before this call returns if the pool
-   * can be immediately drained.
-   */
-  virtual void startDrain() PURE;
-
-  /**
-   * Actively drain all existing connection pool connections. This method can be used in cases
-   * where the connection pool is not being destroyed, but the caller wishes to make sure that
-   * all new streams take place on a new connection. For example, when a health check failure
-   * occurs.
+   * Drains the connections in a pool.
+   * @param drain_behavior A DrainBehavior that controls the behavior of the draining.
    */
-  virtual void drainConnections() PURE;
+  virtual void drainConnections(DrainBehavior drain_behavior) PURE;
 
   /**
    * @return Upstream::HostDescriptionConstSharedPtr the host for which connections are pooled.

envoy/upstream/load_balancer.h

@@ -83,6 +83,25 @@ class LoadBalancerContext {
    * Returns the transport socket options which should be applied on upstream connections
    */
   virtual Network::TransportSocketOptionsConstSharedPtr upstreamTransportSocketOptions() const PURE;
+
+  // Using uint32_t to express expected status of override host. Every bit in the OverrideHostStatus
+  // represent an enum value of Host::Health. The specific correspondence is shown below:
+  //
+  // * 0b001: Host::Health::Unhealthy
+  // * 0b010: Host::Health::Degraded
+  // * 0b100: Host::Health::Healthy
+  //
+  // If multiple bit fields are set, it is acceptable as long as the status of override host is in
+  // any of these statuses.
+  using OverrideHostStatus = uint32_t;
+  using OverrideHost = std::pair<std::string, OverrideHostStatus>;
+
+  /**
+   * Returns the host the load balancer should select directly. If the expected host exists and
+   * the health status of the host matches the expectation, the load balancer can bypass the load
+   * balancing algorithm and return the corresponding host directly.
+   */
+  virtual absl::optional<OverrideHost> overrideHostToSelect() const PURE;
 };
 
 /**