router: add support for retry options predicate extensions

api/envoy/config/route/v3/route_components.proto

@@ -1350,6 +1350,11 @@ message RetryPolicy {
   // details.
   repeated RetryHostPredicate retry_host_predicate = 5;
 
+  // Retry options predicates that will be applied prior to retrying a request. These predicates
+  // allow customizing request behavior between retries.
+  // [#comment: add [#extension-category: envoy.retry_options_predicates] when there are built-in extensions]
+  repeated core.v3.TypedExtensionConfig retry_options_predicates = 12;
+
   // The maximum number of times host selection will be reattempted before giving up, at which
   // point the host that was last selected will be routed to. If unspecified, this will default to
   // retrying once.

docs/root/version_history/current.rst

@@ -115,6 +115,10 @@ New Features
 * overload: add a new overload action that resets streams using a lot of memory. To enable the tracking of allocated bytes in buffers that a stream is using  we need to configure the minimum threshold for tracking via:ref:`buffer_factory_config <envoy_v3_api_field_config.overload.v3.OverloadManager.buffer_factory_config>`. We have an overload action ``Envoy::Server::OverloadActionNameValues::ResetStreams`` that takes advantage of the tracking to  reset the most expensive stream first.
 * rbac: added :ref:`destination_port_range <envoy_v3_api_field_config.rbac.v3.Permission.destination_port_range>` for matching range of destination ports.
 * route config: added :ref:`dynamic_metadata <envoy_v3_api_field_config.route.v3.RouteMatch.dynamic_metadata>` for routing based on dynamic metadata.
+* router: added retry options predicate extensions configured via
+  :ref:` <envoy_v3_api_field_config.route.v3.RetryPolicy.retry_options_predicates>`. These
+  extensions allow modification of requests between retries at the router level. There are not
+  currently any built-in extensions that implement this extension point.
 * router: added :ref:`per_try_idle_timeout <envoy_v3_api_field_config.route.v3.RetryPolicy.per_try_idle_timeout>` timeout configuration.
 * router: added an optional :ref:`override_auto_sni_header <envoy_v3_api_field_config.core.v3.UpstreamHttpProtocolOptions.override_auto_sni_header>` to support setting SNI value from an arbitrary header other than host/authority.
 * sxg_filter: added filter to transform response to SXG package to :ref:`contrib images <install_contrib>`. This can be enabled by setting :ref:`SXG <envoy_v3_api_msg_extensions.filters.http.sxg.v3alpha.SXG>` configuration.

envoy/router/router.h

@@ -236,6 +236,13 @@ class RetryPolicy {
    */
   virtual Upstream::RetryPrioritySharedPtr retryPriority() const PURE;
 
+  /**
+   * @return the retry options predicates for this policy. Each policy will be applied prior
+   * to retrying a request, allowing for request behavior to be customized.
+   */
+  virtual absl::Span<const Upstream::RetryOptionsPredicateConstSharedPtr>
+  retryOptionsPredicates() const PURE;
+
   /**
    * Number of times host selection should be reattempted when selecting a host
    * for a retry attempt.

envoy/upstream/cluster_manager.h

@@ -412,6 +412,11 @@ class ClusterManagerFactory {
    * Returns the secret manager.
    */
   virtual Secret::SecretManager& secretManager() PURE;
+
+  /**
+   * Returns the singleton manager.
+   */
+  virtual Singleton::Manager& singletonManager() PURE;
 };
 
 /**

envoy/upstream/retry.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "envoy/config/typed_config.h"
+#include "envoy/singleton/manager.h"
 #include "envoy/upstream/types.h"
 #include "envoy/upstream/upstream.h"
 
@@ -92,13 +93,58 @@ class RetryHostPredicate {
 
 using RetryHostPredicateSharedPtr = std::shared_ptr<RetryHostPredicate>;
 
+/**
+ * A predicate that is applied prior to retrying a request. Each predicate can customize request
+ * behavior prior to the request being retried.
+ */
+class RetryOptionsPredicate {
+public:
+  struct UpdateOptionsParameters {
+    // Stream info for the previous request attempt that is about to be retried.
+    const StreamInfo::StreamInfo& retriable_request_stream_info_;
+    // The current upstream socket options that were used for connection pool selection on the
+    // previous attempt, or the result of an updated set of options from a previously run
+    // retry options predicate.
+    Network::Socket::OptionsSharedPtr current_upstream_socket_options_;
+  };
+
+  struct UpdateOptionsReturn {
+    // New upstream socket options to apply to the next request attempt. If changed, will affect
+    // connection pool selection similar to that which was done for the initial request.
+    absl::optional<Network::Socket::OptionsSharedPtr> new_upstream_socket_options_;
+  };
+
+  virtual ~RetryOptionsPredicate() = default;
+
+  /**
+   * Update request options.
+   * @param parameters supplies the update parameters.
+   * @return the new options to apply. Each option is wrapped in an optional and is only applied
+   *         if valid.
+   */
+  virtual UpdateOptionsReturn updateOptions(const UpdateOptionsParameters& parameters) const PURE;
+};
+
+using RetryOptionsPredicateConstSharedPtr = std::shared_ptr<const RetryOptionsPredicate>;
+
+/**
+ * Context for all retry extensions.
+ */
+class RetryExtensionFactoryContext {
+public:
+  virtual ~RetryExtensionFactoryContext() = default;
+
+  /**
+   * @return Singleton::Manager& the server-wide singleton manager.
+   */
+  virtual Singleton::Manager& singletonManager() PURE;
+};
+
 /**
  * Factory for RetryPriority.
  */
 class RetryPriorityFactory : public Config::TypedFactory {
 public:
-  ~RetryPriorityFactory() override = default;
-
   virtual RetryPrioritySharedPtr
   createRetryPriority(const Protobuf::Message& config,
                       ProtobufMessage::ValidationVisitor& validation_visitor,
@@ -112,13 +158,23 @@ class RetryPriorityFactory : public Config::TypedFactory {
  */
 class RetryHostPredicateFactory : public Config::TypedFactory {
 public:
-  ~RetryHostPredicateFactory() override = default;
-
   virtual RetryHostPredicateSharedPtr createHostPredicate(const Protobuf::Message& config,
                                                           uint32_t retry_count) PURE;
 
   std::string category() const override { return "envoy.retry_host_predicates"; }
 };
 
+/**
+ * Factory for RetryOptionsPredicate.
+ */
+class RetryOptionsPredicateFactory : public Config::TypedFactory {
+public:
+  virtual RetryOptionsPredicateConstSharedPtr
+  createOptionsPredicate(const Protobuf::Message& config,
+                         RetryExtensionFactoryContext& context) PURE;
+
+  std::string category() const override { return "envoy.retry_options_predicates"; }
+};
+
 } // namespace Upstream
 } // namespace Envoy

source/common/http/async_client_impl.cc

@@ -42,7 +42,7 @@ AsyncClientImpl::AsyncClientImpl(Upstream::ClusterInfoConstSharedPtr cluster,
       config_(http_context.asyncClientStatPrefix(), local_info, stats_store, cm, runtime, random,
               std::move(shadow_writer), true, false, false, false, false, {},
               dispatcher.timeSource(), http_context, router_context),
-      dispatcher_(dispatcher) {}
+      dispatcher_(dispatcher), singleton_manager_(cm.clusterManagerFactory().singletonManager()) {}
 
 AsyncClientImpl::~AsyncClientImpl() {
   while (!active_streams_.empty()) {
@@ -81,8 +81,8 @@ AsyncStreamImpl::AsyncStreamImpl(AsyncClientImpl& parent, AsyncClient::StreamCal
       router_(parent.config_),
       stream_info_(Protocol::Http11, parent.dispatcher().timeSource(), nullptr),
       tracing_config_(Tracing::EgressConfig::get()),
-      route_(std::make_shared<RouteImpl>(parent_.cluster_->name(), options.timeout,
-                                         options.hash_policy, options.retry_policy)),
+      route_(std::make_shared<RouteImpl>(parent_, options.timeout, options.hash_policy,
+                                         options.retry_policy)),
       send_xff_(options.send_xff) {
 
   stream_info_.dynamicMetadata().MergeFrom(options.metadata);

source/common/http/async_client_impl.h

@@ -40,6 +40,7 @@
 #include "source/common/router/router.h"
 #include "source/common/stream_info/stream_info_impl.h"
 #include "source/common/tracing/http_tracer_impl.h"
+#include "source/common/upstream/retry_factory.h"
 
 namespace Envoy {
 namespace Http {
@@ -67,6 +68,7 @@ class AsyncClientImpl final : public AsyncClient {
   Router::FilterConfig config_;
   Event::Dispatcher& dispatcher_;
   std::list<std::unique_ptr<AsyncStreamImpl>> active_streams_;
+  Singleton::Manager& singleton_manager_;
 
   friend class AsyncStreamImpl;
   friend class AsyncRequestImpl;
@@ -124,48 +126,6 @@ class AsyncStreamImpl : public AsyncClient::Stream,
         rate_limit_policy_entry_;
   };
 
-  struct NullRetryPolicy : public Router::RetryPolicy {
-    // Router::RetryPolicy
-    std::chrono::milliseconds perTryTimeout() const override {
-      return std::chrono::milliseconds(0);
-    }
-    std::chrono::milliseconds perTryIdleTimeout() const override {
-      return std::chrono::milliseconds(0);
-    }
-    std::vector<Upstream::RetryHostPredicateSharedPtr> retryHostPredicates() const override {
-      return {};
-    }
-    Upstream::RetryPrioritySharedPtr retryPriority() const override { return {}; }
-
-    uint32_t hostSelectionMaxAttempts() const override { return 1; }
-    uint32_t numRetries() const override { return 1; }
-    uint32_t retryOn() const override { return 0; }
-    const std::vector<uint32_t>& retriableStatusCodes() const override {
-      return retriable_status_codes_;
-    }
-    const std::vector<Http::HeaderMatcherSharedPtr>& retriableHeaders() const override {
-      return retriable_headers_;
-    }
-    const std::vector<Http::HeaderMatcherSharedPtr>& retriableRequestHeaders() const override {
-      return retriable_request_headers_;
-    }
-    absl::optional<std::chrono::milliseconds> baseInterval() const override {
-      return absl::nullopt;
-    }
-    absl::optional<std::chrono::milliseconds> maxInterval() const override { return absl::nullopt; }
-    const std::vector<Router::ResetHeaderParserSharedPtr>& resetHeaders() const override {
-      return reset_headers_;
-    }
-    std::chrono::milliseconds resetMaxInterval() const override {
-      return std::chrono::milliseconds(300000);
-    }
-
-    const std::vector<uint32_t> retriable_status_codes_{};
-    const std::vector<Http::HeaderMatcherSharedPtr> retriable_headers_{};
-    const std::vector<Http::HeaderMatcherSharedPtr> retriable_request_headers_{};
-    const std::vector<Router::ResetHeaderParserSharedPtr> reset_headers_{};
-  };
-
   struct NullConfig : public Router::Config {
     Router::RouteConstSharedPtr route(const Http::RequestHeaderMap&, const StreamInfo::StreamInfo&,
                                       uint64_t) const override {
@@ -211,20 +171,21 @@ class AsyncStreamImpl : public AsyncClient::Stream,
 
   struct RouteEntryImpl : public Router::RouteEntry {
     RouteEntryImpl(
-        const std::string& cluster_name, const absl::optional<std::chrono::milliseconds>& timeout,
+        AsyncClientImpl& parent, const absl::optional<std::chrono::milliseconds>& timeout,
         const Protobuf::RepeatedPtrField<envoy::config::route::v3::RouteAction::HashPolicy>&
             hash_policy,
         const absl::optional<envoy::config::route::v3::RetryPolicy>& retry_policy)
-        : cluster_name_(cluster_name), timeout_(timeout) {
+        : cluster_name_(parent.cluster_->name()), timeout_(timeout) {
       if (!hash_policy.empty()) {
         hash_policy_ = std::make_unique<HashPolicyImpl>(hash_policy);
       }
       if (retry_policy.has_value()) {
         // ProtobufMessage::getStrictValidationVisitor() ?  how often do we do this?
+        Upstream::RetryExtensionFactoryContextImpl factory_context(parent.singleton_manager_);
         retry_policy_ = std::make_unique<Router::RetryPolicyImpl>(
-            retry_policy.value(), ProtobufMessage::getNullValidationVisitor());
+            retry_policy.value(), ProtobufMessage::getNullValidationVisitor(), factory_context);
       } else {
-        retry_policy_ = std::make_unique<NullRetryPolicy>();
+        retry_policy_ = std::make_unique<Router::RetryPolicyImpl>();
       }
     }
 
@@ -330,12 +291,11 @@ class AsyncStreamImpl : public AsyncClient::Stream,
   };
 
   struct RouteImpl : public Router::Route {
-    RouteImpl(const std::string& cluster_name,
-              const absl::optional<std::chrono::milliseconds>& timeout,
+    RouteImpl(AsyncClientImpl& parent, const absl::optional<std::chrono::milliseconds>& timeout,
               const Protobuf::RepeatedPtrField<envoy::config::route::v3::RouteAction::HashPolicy>&
                   hash_policy,
               const absl::optional<envoy::config::route::v3::RetryPolicy>& retry_policy)
-        : route_entry_(cluster_name, timeout, hash_policy, retry_policy), typed_metadata_({}) {}
+        : route_entry_(parent, timeout, hash_policy, retry_policy), typed_metadata_({}) {}
 
     // Router::Route
     const Router::DirectResponseEntry* directResponseEntry() const override { return nullptr; }

source/common/network/BUILD

@@ -343,6 +343,7 @@ envoy_cc_library(
         "//source/common/api:os_sys_calls_lib",
         "//source/common/common:assert_lib",
         "//source/common/common:minimal_logger_lib",
+        "//source/common/common:scalar_to_byte_vector_lib",
         "//source/common/common:utility_lib",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
     ],

source/common/network/addr_family_aware_socket_option_impl.h

@@ -24,9 +24,11 @@ class AddrFamilyAwareSocketOptionImpl : public Socket::Option,
   // Socket::Option
   bool setOption(Socket& socket,
                  envoy::config::core::v3::SocketOption::SocketState state) const override;
-  // The common socket options don't require a hash key.
-  void hashKey(std::vector<uint8_t>&) const override {}
-
+  void hashKey(std::vector<uint8_t>& hash_key) const override {
+    // Add both sub-options to the hash.
+    ipv4_option_->hashKey(hash_key);
+    ipv6_option_->hashKey(hash_key);
+  }
   absl::optional<Details>
   getOptionDetails(const Socket& socket,
                    envoy::config::core::v3::SocketOption::SocketState state) const override;

source/common/network/socket_option_impl.cc

@@ -5,6 +5,7 @@
 
 #include "source/common/api/os_sys_calls_impl.h"
 #include "source/common/common/assert.h"
+#include "source/common/common/scalar_to_byte_vector.h"
 #include "source/common/common/utility.h"
 #include "source/common/network/address_impl.h"
 
@@ -32,6 +33,14 @@ bool SocketOptionImpl::setOption(Socket& socket,
   return true;
 }
 
+void SocketOptionImpl::hashKey(std::vector<uint8_t>& hash_key) const {
+  if (optname_.hasValue()) {
+    pushScalarToByteVector(optname_.level(), hash_key);
+    pushScalarToByteVector(optname_.option(), hash_key);
+    hash_key.insert(hash_key.end(), value_.begin(), value_.end());
+  }
+}
+
 absl::optional<Socket::Option::Details>
 SocketOptionImpl::getOptionDetails(const Socket&,
                                    envoy::config::core::v3::SocketOption::SocketState state) const {

source/common/network/socket_option_impl.h

@@ -134,10 +134,7 @@ class SocketOptionImpl : public Socket::Option, Logger::Loggable<Logger::Id::con
   // Socket::Option
   bool setOption(Socket& socket,
                  envoy::config::core::v3::SocketOption::SocketState state) const override;
-
-  // The common socket options don't require a hash key.
-  void hashKey(std::vector<uint8_t>&) const override {}
-
+  void hashKey(std::vector<uint8_t>& hash_key) const override;
   absl::optional<Details>
   getOptionDetails(const Socket& socket,
                    envoy::config::core::v3::SocketOption::SocketState state) const override;

source/common/network/win32_redirect_records_option_impl.h

@@ -20,8 +20,6 @@ class Win32RedirectRecordsOptionImpl : public Socket::Option,
   // Socket::Option
   bool setOption(Socket& socket,
                  envoy::config::core::v3::SocketOption::SocketState state) const override;
-
-  // The common socket options don't require a hash key.
   void hashKey(std::vector<uint8_t>&) const override;
 
   absl::optional<Details>

source/common/router/BUILD

@@ -63,6 +63,7 @@ envoy_cc_library(
         "//source/common/http:utility_lib",
         "//source/common/protobuf:utility_lib",
         "//source/common/tracing:http_tracer_lib",
+        "//source/common/upstream:retry_factory_lib",
         "//source/extensions/filters/http/common:utility_lib",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/route/v3:pkg_cc_proto",