diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 70c9dd9755c88..26e82df64b01a 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -555,6 +555,7 @@ class PrioritySet { COUNTER(upstream_cx_http2_total) \ COUNTER(upstream_cx_idle_timeout) \ COUNTER(upstream_cx_max_requests) \ + COUNTER(upstream_cx_max_duration) \ COUNTER(upstream_cx_none_healthy) \ COUNTER(upstream_cx_overflow) \ COUNTER(upstream_cx_pool_overflow) \ @@ -706,6 +707,11 @@ class ClusterInfo { */ virtual const absl::optional idleTimeout() const PURE; + /** + * @return the max duration for upstream connection pool connections. + */ + virtual const absl::optional maxConnectionDuration() const PURE; + /** * @return soft limit on size of the cluster's connections read and write buffers. */ diff --git a/source/common/conn_pool/conn_pool_base.cc b/source/common/conn_pool/conn_pool_base.cc index 3c2c0e648db5e..99850e30d0f2c 100644 --- a/source/common/conn_pool/conn_pool_base.cc +++ b/source/common/conn_pool/conn_pool_base.cc @@ -391,7 +391,11 @@ ActiveClient::ActiveClient(ConnPoolImplBase& parent, uint64_t lifetime_request_l conn_length_ = std::make_unique( parent_.host()->cluster().stats().upstream_cx_length_ms_, parent_.dispatcher().timeSource()); connect_timer_->enableTimer(parent_.host()->cluster().connectTimeout()); - + const auto max_connection_duration = parent_.host()->cluster().maxConnectionDuration(); + if (max_connection_duration) { + lifetime_timer_ = parent_.dispatcher().createTimer([this]() -> void { onLifetimeTimeout(); }); + lifetime_timer_->enableTimer(max_connection_duration.value()); + } parent_.host()->stats().cx_total_.inc(); parent_.host()->stats().cx_active_.inc(); parent_.host()->cluster().stats().upstream_cx_total_.inc(); @@ -424,5 +428,14 @@ void ActiveClient::onConnectTimeout() { close(); } +void ActiveClient::onLifetimeTimeout() { + if (state_ != ActiveClient::State::CLOSED) { + ENVOY_CONN_LOG(debug, "lifetime timeout, DRAINING", *this); + parent_.host()->cluster().stats().upstream_cx_max_duration_.inc(); + parent_.transitionActiveClientState(*this, + Envoy::ConnectionPool::ActiveClient::State::DRAINING); + } +} + } // namespace ConnectionPool } // namespace Envoy diff --git a/source/common/conn_pool/conn_pool_base.h b/source/common/conn_pool/conn_pool_base.h index 2488542110e04..588b74b5e52d8 100644 --- a/source/common/conn_pool/conn_pool_base.h +++ b/source/common/conn_pool/conn_pool_base.h @@ -42,6 +42,10 @@ class ActiveClient : public LinkedObject, // Called if the connection does not complete within the cluster's connectTimeout() void onConnectTimeout(); + // Called if the maximum connection duration is reached. If set, this puts an upper + // bound on the lifetime of any connection. + void onLifetimeTimeout(); + // Returns the concurrent request limit, accounting for if the total request limit // is less than the concurrent request limit. uint64_t effectiveConcurrentRequestLimit() const { @@ -74,6 +78,7 @@ class ActiveClient : public LinkedObject, Stats::TimespanPtr conn_connect_ms_; Stats::TimespanPtr conn_length_; Event::TimerPtr connect_timer_; + Event::TimerPtr lifetime_timer_; bool resources_released_{false}; bool timed_out_{false}; }; diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 7967a7d1ba96e..28e85ef49312b 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -789,6 +789,16 @@ ClusterInfoImpl::ClusterInfoImpl( idle_timeout_ = std::chrono::hours(1); } + if (config.common_http_protocol_options().has_max_connection_duration()) { + max_connection_duration_ = std::chrono::milliseconds(DurationUtil::durationToMilliseconds( + config.common_http_protocol_options().max_connection_duration())); + if (max_connection_duration_.value().count() == 0) { + max_connection_duration_ = absl::nullopt; + } + } else { + max_connection_duration_ = absl::nullopt; + } + if (config.has_eds_cluster_config()) { if (config.type() != envoy::config::cluster::v3::Cluster::EDS) { throw EnvoyException("eds_cluster_config set in a non-EDS cluster"); diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 5cdb994b3f416..1804727c6be8f 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -534,6 +534,9 @@ class ClusterInfoImpl : public ClusterInfo, protected Logger::Loggable idleTimeout() const override { return idle_timeout_; } + const absl::optional maxConnectionDuration() const override { + return max_connection_duration_; + } uint32_t perConnectionBufferLimitBytes() const override { return per_connection_buffer_limit_bytes_; } @@ -629,6 +632,7 @@ class ClusterInfoImpl : public ClusterInfo, protected Logger::Loggable idle_timeout_; + absl::optional max_connection_duration_; const uint32_t per_connection_buffer_limit_bytes_; TransportSocketMatcherPtr socket_matcher_; Stats::ScopePtr stats_scope_; diff --git a/test/common/http/http1/conn_pool_test.cc b/test/common/http/http1/conn_pool_test.cc index 91e6925279287..65ad59f458e39 100644 --- a/test/common/http/http1/conn_pool_test.cc +++ b/test/common/http/http1/conn_pool_test.cc @@ -69,6 +69,7 @@ class ConnPoolImplForTest : public ConnPoolImpl { Network::MockClientConnection* connection_; CodecClient* codec_client_; Event::MockTimer* connect_timer_; + Event::MockTimer* lifetime_timer_; Event::DispatcherPtr client_dispatcher_; }; diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc index 168395895ec72..56863a82a07e7 100644 --- a/test/mocks/upstream/cluster_info.cc +++ b/test/mocks/upstream/cluster_info.cc @@ -36,6 +36,13 @@ MockIdleTimeEnabledClusterInfo::MockIdleTimeEnabledClusterInfo() { MockIdleTimeEnabledClusterInfo::~MockIdleTimeEnabledClusterInfo() = default; +MockMaxConnectionDurationEnabledClusterInfo::MockMaxConnectionDurationEnabledClusterInfo() { + ON_CALL(*this, maxConnectionDuration()).WillByDefault(Return(std::chrono::milliseconds(1000))); +} + +MockMaxConnectionDurationEnabledClusterInfo::~MockMaxConnectionDurationEnabledClusterInfo() = + default; + MockClusterInfo::MockClusterInfo() : http2_options_(::Envoy::Http2::Utility::initializeAndValidateOptions( envoy::config::core::v3::Http2ProtocolOptions())), @@ -51,6 +58,8 @@ MockClusterInfo::MockClusterInfo() circuit_breakers_stats_, absl::nullopt, absl::nullopt)) { ON_CALL(*this, connectTimeout()).WillByDefault(Return(std::chrono::milliseconds(1))); ON_CALL(*this, idleTimeout()).WillByDefault(Return(absl::optional())); + ON_CALL(*this, maxConnectionDuration()) + .WillByDefault(Return(absl::optional())); ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, edsServiceName()).WillByDefault(ReturnPointee(&eds_service_name_)); ON_CALL(*this, http1Settings()).WillByDefault(ReturnRef(http1_settings_)); diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index f8bbe8363a818..b204fa555abd6 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -89,6 +89,7 @@ class MockClusterInfo : public ClusterInfo { MOCK_METHOD(bool, addedViaApi, (), (const)); MOCK_METHOD(std::chrono::milliseconds, connectTimeout, (), (const)); MOCK_METHOD(const absl::optional, idleTimeout, (), (const)); + MOCK_METHOD(const absl::optional, maxConnectionDuration, (), (const)); MOCK_METHOD(uint32_t, perConnectionBufferLimitBytes, (), (const)); MOCK_METHOD(uint64_t, features, (), (const)); MOCK_METHOD(const Http::Http1Settings&, http1Settings, (), (const)); @@ -181,5 +182,11 @@ class MockIdleTimeEnabledClusterInfo : public MockClusterInfo { ~MockIdleTimeEnabledClusterInfo() override; }; +class MockMaxConnectionDurationEnabledClusterInfo : public MockClusterInfo { +public: + MockMaxConnectionDurationEnabledClusterInfo(); + ~MockMaxConnectionDurationEnabledClusterInfo() override; +}; + } // namespace Upstream } // namespace Envoy