diff --git a/docs/root/configuration/listeners/stats.rst b/docs/root/configuration/listeners/stats.rst index c4767cc6c0f20..4ad0a29a9c14a 100644 --- a/docs/root/configuration/listeners/stats.rst +++ b/docs/root/configuration/listeners/stats.rst @@ -18,8 +18,10 @@ with the following statistics: downstream_cx_destroy, Counter, Total destroyed connections downstream_cx_active, Gauge, Total active connections downstream_cx_length_ms, Histogram, Connection length milliseconds + downstream_cx_transport_socket_connect_timeout, Counter, Total connections that timed out during transport socket connection negotiation downstream_cx_overflow, Counter, Total connections rejected due to enforcement of listener connection limit downstream_cx_overload_reject, Counter, Total connections rejected due to configured overload actions + downstream_global_cx_overflow, Counter, Total connections rejected due to enforcement of global connection limit downstream_pre_cx_timeout, Counter, Sockets that timed out during listener filter processing downstream_pre_cx_active, Gauge, Sockets currently undergoing listener filter processing global_cx_overflow, Counter, Total connections rejected due to enforcement of the global connection limit diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 357698e616675..72f5ae4ba2b81 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -57,6 +57,7 @@ New Features * http: added :ref:`string_match ` in the header matcher. * http: added support for :ref:`max_requests_per_connection ` for both upstream and downstream connections. * jwt_authn: added support for :ref:`Jwt Cache ` and its size can be specified by :ref:`jwt_cache_size `. +* listener: new listener metric `downstream_cx_transport_socket_connect_timeout` to track transport socket timeouts. * rbac: added :ref:`destination_port_range ` for matching range of destination ports. Deprecated diff --git a/envoy/network/connection.h b/envoy/network/connection.h index 620a5b2e1ec31..da22c5309ab09 100644 --- a/envoy/network/connection.h +++ b/envoy/network/connection.h @@ -339,8 +339,11 @@ class ServerConnection : public virtual Connection { * Set the amount of time allowed for the transport socket to report that a connection is * established. The provided timeout is relative to the current time. If this method is called * after a connection has already been established, it is a no-op. + * + * If a timeout occurs, `timeout_stat` will be incremented. */ - virtual void setTransportSocketConnectTimeout(std::chrono::milliseconds timeout) PURE; + virtual void setTransportSocketConnectTimeout(std::chrono::milliseconds timeout, + Stats::Counter& timeout_stat) PURE; }; using ServerConnectionPtr = std::unique_ptr; diff --git a/source/common/network/connection_impl.cc b/source/common/network/connection_impl.cc index 5325dab742648..558d8f3e0db1a 100644 --- a/source/common/network/connection_impl.cc +++ b/source/common/network/connection_impl.cc @@ -788,10 +788,13 @@ ServerConnectionImpl::ServerConnectionImpl(Event::Dispatcher& dispatcher, : ConnectionImpl(dispatcher, std::move(socket), std::move(transport_socket), stream_info, connected) {} -void ServerConnectionImpl::setTransportSocketConnectTimeout(std::chrono::milliseconds timeout) { +void ServerConnectionImpl::setTransportSocketConnectTimeout(std::chrono::milliseconds timeout, + Stats::Counter& timeout_stat) { if (!transport_connect_pending_) { return; } + + transport_socket_timeout_stat_ = &timeout_stat; if (transport_socket_connect_timer_ == nullptr) { transport_socket_connect_timer_ = dispatcher_.createScaledTimer(Event::ScaledTimerType::TransportSocketConnectTimeout, @@ -814,6 +817,7 @@ void ServerConnectionImpl::raiseEvent(ConnectionEvent event) { void ServerConnectionImpl::onTransportSocketConnectTimeout() { stream_info_.setConnectionTerminationDetails(kTransportSocketConnectTimeoutTerminationDetails); closeConnectionImmediately(); + transport_socket_timeout_stat_->inc(); } ClientConnectionImpl::ClientConnectionImpl( diff --git a/source/common/network/connection_impl.h b/source/common/network/connection_impl.h index 4e9f0044924c1..242d7c926e9dc 100644 --- a/source/common/network/connection_impl.h +++ b/source/common/network/connection_impl.h @@ -217,7 +217,8 @@ class ServerConnectionImpl : public ConnectionImpl, virtual public ServerConnect bool connected); // ServerConnection impl - void setTransportSocketConnectTimeout(std::chrono::milliseconds timeout) override; + void setTransportSocketConnectTimeout(std::chrono::milliseconds timeout, + Stats::Counter& timeout_stat) override; void raiseEvent(ConnectionEvent event) override; private: @@ -227,6 +228,7 @@ class ServerConnectionImpl : public ConnectionImpl, virtual public ServerConnect // Implements a timeout for the transport socket signaling connection. The timer is enabled by a // call to setTransportSocketConnectTimeout and is reset when the connection is established. Event::TimerPtr transport_socket_connect_timer_; + Stats::Counter* transport_socket_timeout_stat_; }; /** diff --git a/source/server/active_listener_base.h b/source/server/active_listener_base.h index 6981900983c89..22dda88adcf78 100644 --- a/source/server/active_listener_base.h +++ b/source/server/active_listener_base.h @@ -11,6 +11,7 @@ namespace Server { COUNTER(downstream_cx_destroy) \ COUNTER(downstream_cx_overflow) \ COUNTER(downstream_cx_total) \ + COUNTER(downstream_cx_transport_socket_connect_timeout) \ COUNTER(downstream_cx_overload_reject) \ COUNTER(downstream_global_cx_overflow) \ COUNTER(downstream_pre_cx_timeout) \ diff --git a/source/server/active_stream_listener_base.cc b/source/server/active_stream_listener_base.cc index 8bec6788040a9..1fe8ec52308b2 100644 --- a/source/server/active_stream_listener_base.cc +++ b/source/server/active_stream_listener_base.cc @@ -47,7 +47,8 @@ void ActiveStreamListenerBase::newConnection(Network::ConnectionSocketPtr&& sock std::move(socket), std::move(transport_socket), *stream_info); if (const auto timeout = filter_chain->transportSocketConnectTimeout(); timeout != std::chrono::milliseconds::zero()) { - server_conn_ptr->setTransportSocketConnectTimeout(timeout); + server_conn_ptr->setTransportSocketConnectTimeout( + timeout, stats_.downstream_cx_transport_socket_connect_timeout_); } server_conn_ptr->setBufferLimits(config_->perConnectionBufferLimitBytes()); RELEASE_ASSERT(server_conn_ptr->addressProvider().remoteAddress() != nullptr, ""); diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 39262ba7164e8..279cefbb0b3bd 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -409,7 +409,9 @@ TEST_P(ConnectionImplTest, SetServerTransportSocketTimeout) { std::move(mocks.transport_socket_), stream_info_, true); EXPECT_CALL(*mock_timer, enableTimer(std::chrono::milliseconds(3 * 1000), _)); - server_connection->setTransportSocketConnectTimeout(std::chrono::seconds(3)); + Stats::MockCounter timeout_counter; + EXPECT_CALL(timeout_counter, inc()); + server_connection->setTransportSocketConnectTimeout(std::chrono::seconds(3), timeout_counter); EXPECT_CALL(*transport_socket, closeSocket(ConnectionEvent::LocalClose)); mock_timer->invokeCallback(); EXPECT_THAT(stream_info_.connectionTerminationDetails(), @@ -429,7 +431,9 @@ TEST_P(ConnectionImplTest, SetServerTransportSocketTimeoutAfterConnect) { transport_socket->callbacks_->raiseEvent(ConnectionEvent::Connected); // This should be a no-op. No timer should be created. EXPECT_CALL(*mocks.dispatcher_, createTimer_(_)).Times(0); - server_connection->setTransportSocketConnectTimeout(std::chrono::seconds(3)); + Stats::MockCounter timeout_counter; + EXPECT_CALL(timeout_counter, inc()).Times(0); + server_connection->setTransportSocketConnectTimeout(std::chrono::seconds(3), timeout_counter); server_connection->close(ConnectionCloseType::NoFlush); } @@ -452,7 +456,9 @@ TEST_P(ConnectionImplTest, ServerTransportSocketTimeoutDisabledOnConnect) { mock_timer->timer_destroyed_ = &timer_destroyed; EXPECT_CALL(*mock_timer, enableTimer(std::chrono::milliseconds(3 * 1000), _)); - server_connection->setTransportSocketConnectTimeout(std::chrono::seconds(3)); + Stats::MockCounter timeout_counter; + EXPECT_CALL(timeout_counter, inc()).Times(0); + server_connection->setTransportSocketConnectTimeout(std::chrono::seconds(3), timeout_counter); transport_socket->callbacks_->raiseEvent(ConnectionEvent::Connected); EXPECT_TRUE(timer_destroyed); diff --git a/test/mocks/network/connection.h b/test/mocks/network/connection.h index 3ec78cb5d47b1..585ef50e2a174 100644 --- a/test/mocks/network/connection.h +++ b/test/mocks/network/connection.h @@ -102,7 +102,7 @@ class MockServerConnection : public ServerConnection, public MockConnectionBase DEFINE_MOCK_CONNECTION_MOCK_METHODS; // Network::ServerConnection - MOCK_METHOD(void, setTransportSocketConnectTimeout, (std::chrono::milliseconds)); + MOCK_METHOD(void, setTransportSocketConnectTimeout, (std::chrono::milliseconds, Stats::Counter&)); }; /** diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index c911742fa469e..f4573005924e9 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -557,7 +557,7 @@ TEST_F(ConnectionHandlerTest, SetsTransportSocketConnectTimeout) { EXPECT_CALL(*filter_chain_, transportSocketConnectTimeout) .WillOnce(Return(std::chrono::seconds(5))); EXPECT_CALL(*server_connection, - setTransportSocketConnectTimeout(std::chrono::milliseconds(5 * 1000))); + setTransportSocketConnectTimeout(std::chrono::milliseconds(5 * 1000), _)); EXPECT_CALL(*access_log_, log(_, _, _, _)); listener_callbacks->onAccept(std::make_unique>());