From cb827119dd828f0d349197f73e32d59cffad918e Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 20 May 2021 10:51:56 -0700 Subject: [PATCH 01/23] initial internal listener Signed-off-by: Yuchen Dai --- include/envoy/event/dispatcher.h | 7 + include/envoy/network/listener.h | 45 + source/common/event/BUILD | 3 + source/common/event/dispatcher_impl.cc | 53 + source/common/event/dispatcher_impl.h | 6 + source/common/network/BUILD | 1 + source/common/network/connection_impl.cc | 40 +- source/common/network/connection_impl.h | 8 + source/common/network/socket_impl.cc | 17 + source/common/network/utility.cc | 3 + source/common/runtime/runtime_features.cc | 1 + source/common/tcp_proxy/tcp_proxy.cc | 9 + source/common/tcp_proxy/tcp_proxy.h | 2 +- source/common/upstream/upstream_impl.cc | 4 +- source/extensions/io_socket/user_space/BUILD | 2 + .../io_socket/user_space/io_handle_impl.cc | 17 +- .../io_socket/user_space/io_handle_impl.h | 2 + source/server/BUILD | 69 +- source/server/active_internal_listener.cc | 180 ++++ source/server/active_internal_listener.h | 151 +++ source/server/active_stream_socket.cc | 169 ++++ source/server/active_stream_socket.h | 224 +++++ source/server/active_tcp_listener.cc | 238 +---- source/server/active_tcp_listener.h | 209 +--- source/server/admin/admin.h | 3 + source/server/connection_handler_impl.cc | 45 +- source/server/connection_handler_impl.h | 16 +- source/server/listener_impl.cc | 24 +- source/server/listener_impl.h | 17 +- source/server/listener_manager_impl.cc | 12 +- source/server/worker_impl.cc | 1 + test/common/network/BUILD | 1 + test/common/network/connection_impl_test.cc | 43 + test/config/utility.cc | 13 +- .../proxy_protocol_regression_test.cc | 3 + .../proxy_protocol/proxy_protocol_test.cc | 6 + test/integration/BUILD | 46 + test/integration/base_integration_test.h | 7 + test/integration/fake_upstream.h | 4 + test/integration/integration_tcp_client.cc | 46 +- test/integration/integration_tcp_client.h | 5 + .../internal_tcp_proxy_integration_test.cc | 903 ++++++++++++++++++ test/integration/server.cc | 10 + test/integration/server.h | 10 + .../socket_interface_integration_test.cc | 30 +- test/integration/tcp_proxy_integration_test.h | 1 - test/mocks/event/mocks.h | 10 + test/mocks/event/wrapped_dispatcher.h | 9 + test/mocks/network/mocks.h | 1 + test/mocks/server/worker.cc | 4 +- test/per_file_coverage.sh | 2 +- test/server/BUILD | 83 ++ test/server/active_internal_listener_test.cc | 188 ++++ test/server/connection_handler_test.cc | 139 ++- test/server/listener_manager_impl_test.cc | 311 ++++++ 55 files changed, 3054 insertions(+), 399 deletions(-) create mode 100644 source/server/active_internal_listener.cc create mode 100644 source/server/active_internal_listener.h create mode 100644 source/server/active_stream_socket.cc create mode 100644 source/server/active_stream_socket.h create mode 100644 test/integration/internal_tcp_proxy_integration_test.cc create mode 100644 test/server/active_internal_listener_test.cc diff --git a/include/envoy/event/dispatcher.h b/include/envoy/event/dispatcher.h index 1618c9236d178..fcdbbcecef0f3 100644 --- a/include/envoy/event/dispatcher.h +++ b/include/envoy/event/dispatcher.h @@ -202,6 +202,13 @@ class Dispatcher : public DispatcherBase { Network::Address::InstanceConstSharedPtr source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) PURE; + /** + * Register an internal listener manager for this dispatcher. + */ + virtual void + registerInternalListenerManager(Network::InternalListenerManager& internal_listener_manager) PURE; + + virtual Network::InternalListenerManagerOptRef getInternalListenerManagerForTest() PURE; /** * Creates an async DNS resolver. The resolver should only be used on the thread that runs this diff --git a/include/envoy/network/listener.h b/include/envoy/network/listener.h index 7c6eabe3e3c6c..6f7850a2e2bbe 100644 --- a/include/envoy/network/listener.h +++ b/include/envoy/network/listener.h @@ -90,6 +90,16 @@ class UdpListenerConfig { using UdpListenerConfigOptRef = OptRef; +/** + * Configuration for an internal listener. + */ +class InternalListenerConfig { +public: + virtual ~InternalListenerConfig() = default; +}; + +using InternalListenerConfigOptRef = OptRef; + /** * A configuration for an individual listener. */ @@ -168,6 +178,11 @@ class ListenerConfig { */ virtual UdpListenerConfigOptRef udpListenerConfig() PURE; + /** + * @return the internal configuration for the listener IFF it is an internal listener. + */ + virtual InternalListenerConfigOptRef internalListenerConfig() PURE; + /** * @return traffic direction of the listener. */ @@ -406,6 +421,36 @@ class UdpListener : public virtual Listener { using UdpListenerPtr = std::unique_ptr; +class InternalListenerCallbacks { +public: + virtual ~InternalListenerCallbacks() = default; + + /** + * Called when a new connection is accepted. + * @param socket supplies the socket that is moved into the callee. + */ + virtual void onAccept(ConnectionSocketPtr&& socket) PURE; + + virtual Event::Dispatcher& dispatcher() PURE; +}; +using InternalListenerCallbacksOptRef = + absl::optional>; + +class InternalListener {}; + +using InternalListenerPtr = std::unique_ptr; +using InternalListenerOptRef = absl::optional>; + +class InternalListenerManager { +public: + virtual ~InternalListenerManager() = default; + virtual InternalListenerCallbacksOptRef + findByAddress(const Address::InstanceConstSharedPtr& listen_address) PURE; +}; + +using InternalListenerManagerOptRef = + absl::optional>; + /** * Handles delivering datagrams to the correct worker. */ diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 333755eeac351..2c5f3b796b529 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -46,6 +46,9 @@ envoy_cc_library( "//source/common/network:dns_lib", "//source/common/network:connection_lib", "//source/common/network:listener_lib", + + #internal address + "//source/extensions/io_socket/user_space:config", ] + select({ "//bazel:apple": ["//source/common/network:apple_dns_lib"], "//conditions:default": [], diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index b3ff5128eb781..07c615a08587c 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -27,6 +27,9 @@ #include "common/network/udp_listener_impl.h" #include "common/runtime/runtime_features.h" +// internal address +#include "extensions/io_socket/user_space/io_handle_impl.h" + #include "event2/event.h" #ifdef ENVOY_HANDLE_SIGNALS @@ -150,7 +153,47 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr Network::Address::InstanceConstSharedPtr source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) { + FANCY_LOG(info, "lambdai: create client connection to {} from {}", + address != nullptr ? address->asStringView() : "nullptr", + source_address != nullptr ? source_address->asStringView() : "nullptr"); ASSERT(isThreadSafe()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.internal_address") && + address->type() == Network::Address::Type::EnvoyInternal) { + Network::IoHandlePtr io_handle_client; + Network::IoHandlePtr io_handle_server; + + std::tie(io_handle_client, io_handle_server) = + Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); + + auto client_conn = std::make_unique( + *this, std::move(io_handle_client), address, source_address, std::move(transport_socket), + options); + // TODO(lambdai): refactor nested if. + auto internal_listener_manager = getInternalListenerManagerForTest(); + if (internal_listener_manager.has_value()) { + // It's either in main thread or the worker is not yet started. + auto internal_listener = internal_listener_manager.value().get().findByAddress(address); + if (internal_listener.has_value()) { + auto accepted_socket = std::make_unique( + std::move(io_handle_server), address, source_address); + // TODO: also check if disabled + internal_listener.value().get().onAccept(std::move(accepted_socket)); + FANCY_LOG(info, "lambdai: find internal listener {} ", address->asStringView()); + } else { + FANCY_LOG(info, "lambdai: cannot find internal listener {} from internal listener manager", + address->asStringView()); + // injected error into client_conn; + io_handle_server->close(); + } + } else { + FANCY_LOG(info, "lambdai: cannot find from internal listener manager while connecting to {}", + address->asStringView()); + // injected error into client_conn; + io_handle_server->close(); + } + + return client_conn; + } return std::make_unique(*this, address, source_address, std::move(transport_socket), options); } @@ -214,6 +257,16 @@ DispatcherImpl::createUdpListener(Network::SocketSharedPtr socket, config); } +void DispatcherImpl::registerInternalListenerManager( + Network::InternalListenerManager& internal_listener_manager) { + ASSERT(!internal_listener_manager_.has_value()); + internal_listener_manager_ = internal_listener_manager; +} + +Network::InternalListenerManagerOptRef DispatcherImpl::getInternalListenerManagerForTest() { + return internal_listener_manager_; +} + TimerPtr DispatcherImpl::createTimer(TimerCb cb) { ASSERT(isThreadSafe()); return createTimerInternal(cb); diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index fe16145e7cd2d..3add83447e3f2 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -78,6 +78,11 @@ class DispatcherImpl : Logger::Loggable, Network::UdpListenerPtr createUdpListener(Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb, const envoy::config::core::v3::UdpSocketConfig& config) override; + + void registerInternalListenerManager( + Network::InternalListenerManager& internal_listener_manager) override; + Network::InternalListenerManagerOptRef getInternalListenerManagerForTest() override; + TimerPtr createTimer(TimerCb cb) override; TimerPtr createScaledTimer(ScaledTimerType timer_type, TimerCb cb) override; TimerPtr createScaledTimer(ScaledTimerMinimum minimum, TimerCb cb) override; @@ -175,6 +180,7 @@ class DispatcherImpl : Logger::Loggable, MonotonicTime approximate_monotonic_time_; WatchdogRegistrationPtr watchdog_registration_; const ScaledRangeTimerManagerPtr scaled_timer_manager_; + Network::InternalListenerManagerOptRef internal_listener_manager_; }; } // namespace Event diff --git a/source/common/network/BUILD b/source/common/network/BUILD index 4991af1ad8996..b1825b10f57e2 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -62,6 +62,7 @@ envoy_cc_library( hdrs = ["connection_impl_base.h"], deps = [ ":filter_manager_lib", + ":listen_socket_lib", "//include/envoy/common:scope_tracker_interface", "//include/envoy/event:dispatcher_interface", "//source/common/common:assert_lib", diff --git a/source/common/network/connection_impl.cc b/source/common/network/connection_impl.cc index 2d964ac4c22a6..59f20a109b079 100644 --- a/source/common/network/connection_impl.cc +++ b/source/common/network/connection_impl.cc @@ -21,6 +21,7 @@ #include "common/network/listen_socket_impl.h" #include "common/network/raw_buffer_socket.h" #include "common/network/utility.h" +#include "common/runtime/runtime_features.h" namespace Envoy { namespace Network { @@ -274,7 +275,8 @@ void ConnectionImpl::noDelay(bool enable) { } // Don't set NODELAY for unix domain sockets - if (socket_->addressType() == Address::Type::Pipe) { + if (socket_->addressType() == Address::Type::Pipe || + socket_->addressType() == Address::Type::EnvoyInternal) { return; } @@ -657,9 +659,16 @@ void ConnectionImpl::onWriteReady() { if (connecting_) { int error; - socklen_t error_size = sizeof(error); - RELEASE_ASSERT(socket_->getSocketOption(SOL_SOCKET, SO_ERROR, &error, &error_size).rc_ == 0, - ""); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.internal_address") && + socket_->addressProvider().remoteAddress()->type() == + Network::Address::Type::EnvoyInternal) { + // Connected! + error = 0; + } else { + socklen_t error_size = sizeof(error); + RELEASE_ASSERT(socket_->getSocketOption(SOL_SOCKET, SO_ERROR, &error, &error_size).rc_ == 0, + ""); + } if (error == 0) { ENVOY_CONN_LOG(debug, "connected", *this); @@ -810,6 +819,21 @@ void ServerConnectionImpl::onTransportSocketConnectTimeout() { closeConnectionImmediately(); } +ClientConnectionImpl::ClientConnectionImpl( + Event::Dispatcher& dispatcher, std::unique_ptr client_io_handle, + const Network::Address::InstanceConstSharedPtr& address, + const Network::Address::InstanceConstSharedPtr& source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) + : ConnectionImpl( + dispatcher, + std::make_unique(std::move(client_io_handle), nullptr, address), + std::move(transport_socket), stream_info_, false), + stream_info_(dispatcher.timeSource(), socket_->addressProviderSharedPtr()) { + UNREFERENCED_PARAMETER(source_address); + UNREFERENCED_PARAMETER(options); +} + ClientConnectionImpl::ClientConnectionImpl( Event::Dispatcher& dispatcher, const Address::InstanceConstSharedPtr& remote_address, const Network::Address::InstanceConstSharedPtr& source_address, @@ -860,8 +884,12 @@ void ClientConnectionImpl::connect() { socket_->addressProvider().remoteAddress()->asString()); const Api::SysCallIntResult result = socket_->connect(socket_->addressProvider().remoteAddress()); if (result.rc_ == 0) { - // write will become ready. - ASSERT(connecting_); + if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.internal_address") && + socket_->addressProvider().remoteAddress()->type() != + Network::Address::Type::EnvoyInternal) { + // write will become ready. + ASSERT(connecting_); + } } else { ASSERT(SOCKET_FAILURE(result.rc_)); #ifdef WIN32 diff --git a/source/common/network/connection_impl.h b/source/common/network/connection_impl.h index 17021980febbf..90eda7f5397d7 100644 --- a/source/common/network/connection_impl.h +++ b/source/common/network/connection_impl.h @@ -12,6 +12,7 @@ #include "common/buffer/watermark_buffer.h" #include "common/event/libevent.h" #include "common/network/connection_impl_base.h" +#include "common/network/listen_socket_impl.h" #include "common/stream_info/stream_info_impl.h" #include "absl/types/optional.h" @@ -239,6 +240,13 @@ class ClientConnectionImpl : public ConnectionImpl, virtual public ClientConnect const Address::InstanceConstSharedPtr& source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options); + // Internal socket is manufactured by its own factory. + ClientConnectionImpl(Event::Dispatcher& dispatcher, + std::unique_ptr client_io_handle, + const Network::Address::InstanceConstSharedPtr& address, + const Network::Address::InstanceConstSharedPtr& source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options); // Network::ClientConnection void connect() override; diff --git a/source/common/network/socket_impl.cc b/source/common/network/socket_impl.cc index 7a558ca059081..e0c81f5cc39bc 100644 --- a/source/common/network/socket_impl.cc +++ b/source/common/network/socket_impl.cc @@ -22,11 +22,28 @@ SocketImpl::SocketImpl(IoHandlePtr&& io_handle, : io_handle_(std::move(io_handle)), address_provider_(std::make_shared(local_address, remote_address)) { + FANCY_LOG(debug, + "lambdai: SocketImpl::SocketImpl( local address = {} provider local address = {}", + local_address == nullptr ? "nullptr" : local_address->asStringView(), + address_provider_->localAddress() == nullptr + ? "nullptr" + : address_provider_->localAddress()->asStringView()); + FANCY_LOG(debug, + "lambdai: SocketImpl::SocketImpl( remote address = {} provider local address = {}", + remote_address == nullptr ? "nullptr" : remote_address->asStringView(), + address_provider_->remoteAddress() == nullptr + ? "nullptr" + : address_provider_->remoteAddress()->asStringView()); if (address_provider_->localAddress() != nullptr) { addr_type_ = address_provider_->localAddress()->type(); return; } + if (address_provider_->remoteAddress() != nullptr) { + addr_type_ = address_provider_->remoteAddress()->type(); + return; + } + // Should not happen but some tests inject -1 fds if (!io_handle_->isOpen()) { return; diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index d30b5770b63ed..855f5b4cc2cda 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -522,6 +522,9 @@ Utility::protobufAddressSocketType(const envoy::config::core::v3::Address& proto } case envoy::config::core::v3::Address::AddressCase::kPipe: return Socket::Type::Stream; + case envoy::config::core::v3::Address::AddressCase::kEnvoyInternalAddress: + // Currently internal address supports stream operation only. + return Socket::Type::Stream; default: NOT_REACHED_GCOVR_EXCL_LINE; } diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index c44cc2a55ad38..22f1ca7a9c3d9 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -75,6 +75,7 @@ constexpr const char* runtime_features[] = { "envoy.reloadable_features.http_upstream_wait_connect_response", "envoy.reloadable_features.http2_skip_encoding_empty_trailers", "envoy.reloadable_features.improved_stream_limit_handling", + "envoy.reloadable_features.internal_address", "envoy.reloadable_features.internal_redirects_with_body", "envoy.reloadable_features.new_tcp_connection_pool", "envoy.reloadable_features.prefer_quic_kernel_bpf_packet_routing", diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index fb76f6a942c7e..d0a0fc7fa4a74 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -589,6 +589,8 @@ Network::FilterStatus Filter::onNewConnection() { } void Filter::onDownstreamEvent(Network::ConnectionEvent event) { + ENVOY_LOG(debug, "lambdai: tcp proxy filter on downstream event {}, upstream_ = {}", + static_cast(event), static_cast(upstream_.get())); if (upstream_) { Tcp::ConnectionPool::ConnectionDataPtr conn_data(upstream_->onDownstreamEvent(event)); if (conn_data != nullptr && @@ -751,6 +753,9 @@ Drainer::Drainer(UpstreamDrainManager& parent, const Config::SharedConfigSharedP timer_(std::move(idle_timer)), upstream_host_(upstream_host), config_(config) { config_->stats().upstream_flush_total_.inc(); config_->stats().upstream_flush_active_.inc(); + ENVOY_CONN_LOG(debug, "set drainer and bump upstream_flush_total_ to {}, active to {} ", + upstream_conn_data_->connection(), config_->stats().upstream_flush_total_.value(), + config_->stats().upstream_flush_active_.value()); } void Drainer::onEvent(Network::ConnectionEvent event) { @@ -760,6 +765,10 @@ void Drainer::onEvent(Network::ConnectionEvent event) { timer_->disableTimer(); } config_->stats().upstream_flush_active_.dec(); + ENVOY_CONN_LOG( + debug, "drainer on close event and set upstream_flush_total_ to {}, active to {}", + upstream_conn_data_->connection(), config_->stats().upstream_flush_total_.value(), + config_->stats().upstream_flush_active_.value()); parent_.remove(*this, upstream_conn_data_->connection().dispatcher()); } } diff --git a/source/common/tcp_proxy/tcp_proxy.h b/source/common/tcp_proxy/tcp_proxy.h index 6579995d1340e..b10aecf1cf5d3 100644 --- a/source/common/tcp_proxy/tcp_proxy.h +++ b/source/common/tcp_proxy/tcp_proxy.h @@ -389,7 +389,7 @@ class Filter : public Network::ReadFilter, // This class deals with an upstream connection that needs to finish flushing, when the downstream // connection has been closed. The TcpProxy is destroyed when the downstream connection is closed, // so handling the upstream connection here allows it to finish draining or timeout. -class Drainer : public Event::DeferredDeletable { +class Drainer : public Event::DeferredDeletable, protected Logger::Loggable { public: Drainer(UpstreamDrainManager& parent, const Config::SharedConfigSharedPtr& config, const std::shared_ptr& callbacks, diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index a4dbd0eeb21a7..1cbeff0b5411f 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -338,7 +338,9 @@ HostImpl::createConnection(Event::Dispatcher& dispatcher, const ClusterInfo& clu } else { connection_options = options; } - ASSERT(!address->envoyInternalAddress()); + if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.internal_address")) { + ASSERT(!address->envoyInternalAddress()); + } Network::ClientConnectionPtr connection = dispatcher.createClientConnection( address, cluster.sourceAddress(), socket_factory.createTransportSocket(std::move(transport_socket_options)), diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 18a01f6e1eaec..a070a03e1039e 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -16,7 +16,9 @@ envoy_cc_extension( security_posture = "unknown", status = "wip", undocumented = True, + visibility = ["//visibility:public"], deps = [ + ":io_handle_impl_lib", ], ) diff --git a/source/extensions/io_socket/user_space/io_handle_impl.cc b/source/extensions/io_socket/user_space/io_handle_impl.cc index 4c487cf5fa9e6..495aa8bf10ac2 100644 --- a/source/extensions/io_socket/user_space/io_handle_impl.cc +++ b/source/extensions/io_socket/user_space/io_handle_impl.cc @@ -275,10 +275,19 @@ Network::IoHandlePtr IoHandleImpl::accept(struct sockaddr*, socklen_t*) { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } -Api::SysCallIntResult IoHandleImpl::connect(Network::Address::InstanceConstSharedPtr) { - // Buffered Io handle should always be considered as connected. - // Use write or read to determine if peer is closed. - return {0, 0}; +Api::SysCallIntResult IoHandleImpl::connect(Network::Address::InstanceConstSharedPtr address) { + if (peer_handle_ != nullptr) { + FANCY_LOG(info, "lambdai: user handle {} connect to {} has peer_handler, returning 0", + static_cast(this), address->asStringView()); + + // Buffered Io handle should always be considered as connected unless the server peer cannot be + // found. Use write or read to determine if peer is closed. + return {0, 0}; + } else { + FANCY_LOG(info, "lambdai: user handle {} connect to {}. no peer_handler, returning error", + static_cast(this), address->asStringView()); + return Api::SysCallIntResult{-1, SOCKET_ERROR_INVAL}; + } } Api::SysCallIntResult IoHandleImpl::setOption(int, int, const void*, socklen_t) { diff --git a/source/extensions/io_socket/user_space/io_handle_impl.h b/source/extensions/io_socket/user_space/io_handle_impl.h index d0ceab1d5cf84..d9feb7991d8ed 100644 --- a/source/extensions/io_socket/user_space/io_handle_impl.h +++ b/source/extensions/io_socket/user_space/io_handle_impl.h @@ -144,6 +144,8 @@ class IoHandleImpl final : public Network::IoHandle, ASSERT(!peer_handle_); ASSERT(!write_shutdown_); peer_handle_ = writable_peer; + FANCY_LOG(debug, "lambdai: set iohandle {} set peer_handle_ to {} ", static_cast(this), + static_cast(writable_peer)); } private: diff --git a/source/server/BUILD b/source/server/BUILD index bd71a70146747..2d5f4d6fbb1d6 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -64,9 +64,10 @@ envoy_cc_library( envoy_cc_library( name = "connection_handler_lib", deps = [ - "//source/server:active_tcp_listener", - "//source/server:active_udp_listener", - "//source/server:connection_handler_impl", + ":active_internal_listener", + ":active_tcp_listener", + ":active_udp_listener", + ":connection_handler_impl", ], ) @@ -77,6 +78,7 @@ envoy_cc_library( "connection_handler_impl.h", ], deps = [ + ":active_internal_listener", ":active_tcp_listener", "//include/envoy/common:time_interface", "//include/envoy/event:deferred_deletable", @@ -116,6 +118,7 @@ envoy_cc_library( "active_tcp_listener.h", ], deps = [ + ":active_stream_socket", "//include/envoy/common:time_interface", "//include/envoy/event:deferred_deletable", "//include/envoy/event:dispatcher_interface", @@ -156,6 +159,66 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "active_stream_socket", + srcs = ["active_stream_socket.cc"], + hdrs = [ + "active_stream_socket.h", + ], + deps = [ + ":active_listener_base", + "//include/envoy/common:time_interface", + "//include/envoy/event:deferred_deletable", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/event:timer_interface", + "//include/envoy/network:connection_handler_interface", + "//include/envoy/network:connection_interface", + "//include/envoy/network:exception_interface", + "//include/envoy/network:filter_interface", + "//include/envoy/network:listen_socket_interface", + "//include/envoy/network:listener_interface", + "//include/envoy/server:listener_manager_interface", + "//include/envoy/stats:timespan_interface", + "//source/common/common:linked_object", + "//source/common/common:non_copyable", + "//source/common/event:deferred_task", + "//source/common/network:connection_lib", + "//source/common/stats:timespan_lib", + "//source/common/stream_info:stream_info_lib", + "//source/extensions/transport_sockets:well_known_names", + ], +) + +envoy_cc_library( + name = "active_internal_listener", + srcs = ["active_internal_listener.cc"], + hdrs = [ + "active_internal_listener.h", + ], + deps = [ + ":active_stream_socket", + "//include/envoy/common:time_interface", + "//include/envoy/event:deferred_deletable", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/event:timer_interface", + "//include/envoy/network:connection_handler_interface", + "//include/envoy/network:connection_interface", + "//include/envoy/network:exception_interface", + "//include/envoy/network:filter_interface", + "//include/envoy/network:listen_socket_interface", + "//include/envoy/network:listener_interface", + "//include/envoy/server:listener_manager_interface", + "//include/envoy/stats:timespan_interface", + "//source/common/common:linked_object", + "//source/common/common:non_copyable", + "//source/common/event:deferred_task", + "//source/common/network:connection_lib", + "//source/common/stats:timespan_lib", + "//source/common/stream_info:stream_info_lib", + "//source/extensions/transport_sockets:well_known_names", + ], +) + envoy_cc_library( name = "drain_manager_lib", srcs = ["drain_manager_impl.cc"], diff --git a/source/server/active_internal_listener.cc b/source/server/active_internal_listener.cc new file mode 100644 index 0000000000000..192c5c332e923 --- /dev/null +++ b/source/server/active_internal_listener.cc @@ -0,0 +1,180 @@ +#include "server/active_internal_listener.h" + +#include "envoy/network/filter.h" +#include "envoy/stats/scope.h" + +#include "common/network/address_impl.h" +#include "common/stats/timespan_impl.h" + +#include "extensions/transport_sockets/well_known_names.h" + +namespace Envoy { +namespace Server { + +ActiveInternalListener::ActiveInternalListener(Network::ConnectionHandler& conn_handler, + Event::Dispatcher& dispatcher, + Network::ListenerConfig& config) + : TypedActiveStreamListenerBase( + conn_handler, dispatcher, + std::make_unique(), config) {} +ActiveInternalListener::ActiveInternalListener(Network::ConnectionHandler& conn_handler, + Event::Dispatcher& dispatcher, + Network::ListenerPtr listener, + Network::ListenerConfig& config) + : TypedActiveStreamListenerBase(conn_handler, dispatcher, + std::move(listener), config) {} + +ActiveInternalListener::~ActiveInternalListener() { cleanupConnections(); } + +void ActiveInternalListener::removeConnection(ActiveInternalConnection& connection) { + ENVOY_CONN_LOG(debug, "adding to cleanup list", *connection.connection_); + ActiveInternalConnections& active_connections = connection.active_connections_; + ActiveInternalConnectionPtr removed = connection.removeFromList(active_connections.connections_); + dispatcher().deferredDelete(std::move(removed)); + // Delete map entry only iff connections becomes empty. + if (active_connections.connections_.empty()) { + auto iter = connections_by_context_.find(&active_connections.filter_chain_); + ASSERT(iter != connections_by_context_.end()); + // To cover the lifetime of every single connection, Connections need to be deferred deleted + // because the previously contained connection is deferred deleted. + dispatcher().deferredDelete(std::move(iter->second)); + // The erase will break the iteration over the connections_by_context_ during the deletion. + if (!is_deleting_) { + connections_by_context_.erase(iter); + } + } +} + +void ActiveInternalListener::newConnection(Network::ConnectionSocketPtr&& socket, + std::unique_ptr stream_info) { + + // Find matching filter chain. + const auto filter_chain = config_->filterChainManager().findFilterChain(*socket); + if (filter_chain == nullptr) { + ENVOY_LOG(debug, "closing connection: no matching filter chain found"); + stats_.no_filter_chain_match_.inc(); + stream_info->setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); + stream_info->setResponseCodeDetails(StreamInfo::ResponseCodeDetails::get().FilterChainNotFound); + emitLogs(*config_, *stream_info); + socket->close(); + return; + } + + stream_info->setFilterChainName(filter_chain->name()); + auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); + stream_info->setDownstreamSslConnection(transport_socket->ssl()); + auto& active_connections = getOrCreateActiveConnections(*filter_chain); + auto server_conn_ptr = dispatcher().createServerConnection( + std::move(socket), std::move(transport_socket), *stream_info); + if (const auto timeout = filter_chain->transportSocketConnectTimeout(); + timeout != std::chrono::milliseconds::zero()) { + server_conn_ptr->setTransportSocketConnectTimeout(timeout); + } + ActiveInternalConnectionPtr active_connection( + new ActiveInternalConnection(active_connections, std::move(server_conn_ptr), + dispatcher().timeSource(), std::move(stream_info))); + active_connection->connection_->setBufferLimits(config_->perConnectionBufferLimitBytes()); + + const bool empty_filter_chain = !config_->filterChainFactory().createNetworkFilterChain( + *active_connection->connection_, filter_chain->networkFilterFactories()); + if (empty_filter_chain) { + ENVOY_CONN_LOG(debug, "closing connection: no filters", *active_connection->connection_); + active_connection->connection_->close(Network::ConnectionCloseType::NoFlush); + } + + // If the connection is already closed, we can just let this connection immediately die. + if (active_connection->connection_->state() != Network::Connection::State::Closed) { + ENVOY_CONN_LOG(debug, "new connection", *active_connection->connection_); + active_connection->connection_->addConnectionCallbacks(*active_connection); + LinkedList::moveIntoList(std::move(active_connection), active_connections.connections_); + } +} + +ActiveInternalConnections& +ActiveInternalListener::getOrCreateActiveConnections(const Network::FilterChain& filter_chain) { + ActiveInternalConnectionsPtr& connections = connections_by_context_[&filter_chain]; + if (connections == nullptr) { + connections = std::make_unique(*this, filter_chain); + } + return *connections; +} + +void ActiveInternalListener::updateListenerConfig(Network::ListenerConfig& config) { + ENVOY_LOG(trace, "replacing listener ", config_->listenerTag(), " by ", config.listenerTag()); + config_ = &config; +} + +ActiveInternalConnections::ActiveInternalConnections(ActiveInternalListener& listener, + const Network::FilterChain& filter_chain) + : listener_(listener), filter_chain_(filter_chain) {} + +ActiveInternalConnections::~ActiveInternalConnections() { + // connections should be defer deleted already. + ASSERT(connections_.empty()); +} + +ActiveInternalConnection::ActiveInternalConnection( + ActiveInternalConnections& active_connections, Network::ConnectionPtr&& new_connection, + TimeSource& time_source, std::unique_ptr&& stream_info) + : stream_info_(std::move(stream_info)), active_connections_(active_connections), + connection_(std::move(new_connection)), + conn_length_(new Stats::HistogramCompletableTimespanImpl( + active_connections_.listener_.stats_.downstream_cx_length_ms_, time_source)) { + // We just universally set no delay on connections. Theoretically we might at some point want + // to make this configurable. + connection_->noDelay(true); + auto& listener = active_connections_.listener_; + listener.stats_.downstream_cx_total_.inc(); + listener.stats_.downstream_cx_active_.inc(); + listener.per_worker_stats_.downstream_cx_total_.inc(); + listener.per_worker_stats_.downstream_cx_active_.inc(); + stream_info_->setConnectionID(connection_->id()); + + // Active connections on the handler (not listener). The per listener connections have already + // been incremented at this point either via the connection balancer or in the socket accept + // path if there is no configured balancer. + listener.parent_.incNumConnections(); +} + +ActiveInternalConnection::~ActiveInternalConnection() { + ActiveInternalListener::emitLogs(*active_connections_.listener_.config_, *stream_info_); + auto& listener = active_connections_.listener_; + listener.stats_.downstream_cx_active_.dec(); + listener.stats_.downstream_cx_destroy_.inc(); + listener.per_worker_stats_.downstream_cx_active_.dec(); + conn_length_->complete(); + + // Active listener connections (not handler). + listener.decNumConnections(); + + // Active handler connections (not listener). + listener.parent_.decNumConnections(); +} + +void ActiveInternalListener::onAccept(Network::ConnectionSocketPtr&& socket) { + // Unlike tcp listener, no rebalancer is applied and won't call pickTargetHandler to account + // connections. + incNumConnections(); + + auto active_socket = std::make_unique( + *this, std::move(socket), false /* do not handle off at internal listener */); + // TODO(lambdai): restore address from either socket options, or from listener config. + active_socket->socket_->addressProvider().restoreLocalAddress( + std::make_shared("255.255.255.255", 0)); + active_socket->socket_->addressProvider().setRemoteAddress( + std::make_shared("255.255.255.254", 0)); + + onSocketAccepted(std::move(active_socket)); +} + +// Network::ConnectionCallbacks +void ActiveInternalConnection::onEvent(Network::ConnectionEvent event) { + FANCY_LOG(info, "lambdai: internal connection on event {}", static_cast(event)); + // Any event leads to destruction of the connection. + if (event == Network::ConnectionEvent::LocalClose || + event == Network::ConnectionEvent::RemoteClose) { + active_connections_.listener_.removeConnection(*this); + } +} +} // namespace Server +} // namespace Envoy diff --git a/source/server/active_internal_listener.h b/source/server/active_internal_listener.h new file mode 100644 index 0000000000000..7d9a3d15103a6 --- /dev/null +++ b/source/server/active_internal_listener.h @@ -0,0 +1,151 @@ +#pragma once +#include +#include +#include +#include + +#include "envoy/common/time.h" +#include "envoy/event/deferred_deletable.h" +#include "envoy/event/dispatcher.h" +#include "envoy/network/connection.h" +#include "envoy/network/connection_handler.h" +#include "envoy/network/filter.h" +#include "envoy/network/listen_socket.h" +#include "envoy/network/listener.h" +#include "envoy/server/listener_manager.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/timespan.h" + +#include "common/common/linked_object.h" +#include "common/common/non_copyable.h" +#include "common/stream_info/stream_info_impl.h" + +#include "server/active_stream_socket.h" + +#include "spdlog/spdlog.h" + +namespace Envoy { +namespace Server { + +struct ActiveInternalConnection; +using ActiveInternalConnectionPtr = std::unique_ptr; + +class ActiveInternalConnections; +using ActiveInternalConnectionsPtr = std::unique_ptr; + +class ActiveInternalListener; +/** + * Wrapper for a group of active connections which are attached to the same filter chain context. + */ +class ActiveInternalConnections : public Event::DeferredDeletable { +public: + ActiveInternalConnections(ActiveInternalListener& listener, + const Network::FilterChain& filter_chain); + ~ActiveInternalConnections() override; + + // Listener filter chain pair is the owner of the connections. + ActiveInternalListener& listener_; + const Network::FilterChain& filter_chain_; + // Owned connections. + std::list connections_; +}; + +/** + * Wrapper for an active internal connection owned by this handler. + */ +struct ActiveInternalConnection : LinkedObject, + public Event::DeferredDeletable, + public Network::ConnectionCallbacks { + ActiveInternalConnection(ActiveInternalConnections& active_connections, + Network::ConnectionPtr&& new_connection, TimeSource& time_system, + std::unique_ptr&& stream_info); + ~ActiveInternalConnection() override; + + using CollectionType = ActiveInternalConnections; + + // Network::ConnectionCallbacks + void onEvent(Network::ConnectionEvent event) override; + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} + + std::unique_ptr stream_info_; + ActiveInternalConnections& active_connections_; + Network::ConnectionPtr connection_; + Stats::TimespanPtr conn_length_; +}; + +class ActiveInternalListener : public TypedActiveStreamListenerBase, + public Network::InternalListenerCallbacks, + Logger::Loggable { +public: + ActiveInternalListener(Network::ConnectionHandler& conn_handler, Event::Dispatcher& dispatcher, + Network::ListenerConfig& config); + ActiveInternalListener(Network::ConnectionHandler& conn_handler, Event::Dispatcher& dispatcher, + Network::ListenerPtr listener, Network::ListenerConfig& config); + ~ActiveInternalListener() override; + + class NetworkInternalListener : public Network::Listener { + + void disable() override { + // TODO(lambdai): think about how to elegantly disable internal listener. (Queue socket or + // close socket immediately?) + ENVOY_LOG(debug, "Warning: the internal listener cannot be disabled."); + } + + void enable() override { + ENVOY_LOG(debug, "Warning: the internal listener is always enabled."); + } + + void setRejectFraction(UnitFloat) override {} + }; + // ActiveListenerImplBase + Network::Listener* listener() override { return listener_.get(); } + Network::BalancedConnectionHandlerOptRef + getBalancedHandlerByAddress(const Network::Address::Instance&) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + + void pauseListening() override { + if (listener_ != nullptr) { + listener_->disable(); + } + } + void resumeListening() override { + if (listener_ != nullptr) { + listener_->enable(); + } + } + void shutdownListener() override { listener_.reset(); } + + // Network::InternalListenerCallbacks + void onAccept(Network::ConnectionSocketPtr&& socket) override; + Event::Dispatcher& dispatcher() override { return dispatcher_; } + + void incNumConnections() override { config_->openConnections().inc(); } + void decNumConnections() override { config_->openConnections().dec(); } + /** + * Remove and destroy an active connection. + * @param connection supplies the connection to remove. + */ + void removeConnection(ActiveInternalConnection& connection); + + /** + * Create a new connection from a socket accepted by the listener. + */ + void newConnection(Network::ConnectionSocketPtr&& socket, + std::unique_ptr stream_info) override; + + /** + * Return the active connections container attached with the given filter chain. + */ + ActiveInternalConnections& getOrCreateActiveConnections(const Network::FilterChain& filter_chain); + + /** + * Update the listener config. The follow up connections will see the new config. The existing + * connections are not impacted. + */ + void updateListenerConfig(Network::ListenerConfig& config); +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/active_stream_socket.cc b/source/server/active_stream_socket.cc new file mode 100644 index 0000000000000..660682a1800f1 --- /dev/null +++ b/source/server/active_stream_socket.cc @@ -0,0 +1,169 @@ +#include "server/active_stream_socket.h" + +#include "envoy/network/filter.h" +#include "envoy/stats/scope.h" + +#include "common/stats/timespan_impl.h" + +#include "extensions/transport_sockets/well_known_names.h" + +namespace Envoy { +namespace Server { + +ActiveStreamSocket::ActiveStreamSocket(ActiveStreamListenerBase& listener, + Network::ConnectionSocketPtr&& socket, + bool hand_off_restored_destination_connections) + : listener_(listener), socket_(std::move(socket)), + hand_off_restored_destination_connections_(hand_off_restored_destination_connections), + iter_(accept_filters_.end()), + stream_info_(std::make_unique( + listener_.dispatcher().timeSource(), socket_->addressProviderSharedPtr(), + StreamInfo::FilterState::LifeSpan::Connection)) { + listener_.stats_.downstream_pre_cx_active_.inc(); +} +ActiveStreamSocket::~ActiveStreamSocket() { + accept_filters_.clear(); + listener_.stats_.downstream_pre_cx_active_.dec(); + + // If the underlying socket is no longer attached, it means that it has been transferred to + // an active connection. In this case, the active connection will decrement the number + // of listener connections. + // TODO(mattklein123): In general the way we account for the number of listener connections + // is incredibly fragile. Revisit this by potentially merging ActiveTcpSocket and + // ActiveTcpConnection, having a shared object which does accounting (but would require + // another allocation, etc.). + if (socket_ != nullptr) { + listener_.decNumConnections(); + } +} +Event::Dispatcher& ActiveStreamSocket::dispatcher() { return listener_.dispatcher(); } + +ActiveStreamListenerBase::ActiveStreamListenerBase(Network::ConnectionHandler& parent, + Event::Dispatcher& dispatcher, + Network::ListenerPtr&& listener, + Network::ListenerConfig& config) + : ActiveListenerImplBase(parent, &config), parent_(parent), dispatcher_(dispatcher), + listener_(std::move(listener)), listener_filters_timeout_(config.listenerFiltersTimeout()), + continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()) {} + +void ActiveStreamListenerBase::emitLogs(Network::ListenerConfig& config, + StreamInfo::StreamInfo& stream_info) { + stream_info.onRequestComplete(); + for (const auto& access_log : config.accessLogs()) { + access_log->log(nullptr, nullptr, nullptr, stream_info); + } +} + +void ActiveStreamSocket::onTimeout() { + listener_.stats_.downstream_pre_cx_timeout_.inc(); + ASSERT(inserted()); + ENVOY_LOG(debug, "listener filter times out after {} ms", + listener_.listener_filters_timeout_.count()); + + if (listener_.continue_on_listener_filters_timeout_) { + ENVOY_LOG(debug, "fallback to default listener filter"); + newConnection(); + } + unlink(); +} + +void ActiveStreamSocket::startTimer() { + if (listener_.listener_filters_timeout_.count() > 0) { + timer_ = listener_.dispatcher().createTimer([this]() -> void { onTimeout(); }); + timer_->enableTimer(listener_.listener_filters_timeout_); + } +} + +void ActiveStreamSocket::unlink() { + auto removed = removeFromList(listener_.sockets_); + if (removed->timer_ != nullptr) { + removed->timer_->disableTimer(); + } + // Emit logs if a connection is not established. + if (!connected_ && stream_info_ != nullptr) { + ActiveStreamListenerBase::emitLogs(*listener_.config_, *stream_info_); + } + listener_.dispatcher().deferredDelete(std::move(removed)); +} + +void ActiveStreamSocket::continueFilterChain(bool success) { + if (success) { + bool no_error = true; + if (iter_ == accept_filters_.end()) { + iter_ = accept_filters_.begin(); + } else { + iter_ = std::next(iter_); + } + + for (; iter_ != accept_filters_.end(); iter_++) { + Network::FilterStatus status = (*iter_)->onAccept(*this); + if (status == Network::FilterStatus::StopIteration) { + // The filter is responsible for calling us again at a later time to continue the filter + // chain from the next filter. + if (!socket().ioHandle().isOpen()) { + // break the loop but should not create new connection + no_error = false; + break; + } else { + // Blocking at the filter but no error + return; + } + } + } + // Successfully ran all the accept filters. + if (no_error) { + newConnection(); + } else { + // Signal the caller that no extra filter chain iteration is needed. + iter_ = accept_filters_.end(); + } + } + + // Filter execution concluded, unlink and delete this ActiveTcpSocket if it was linked. + if (inserted()) { + unlink(); + } +} + +void ActiveStreamSocket::setDynamicMetadata(const std::string& name, + const ProtobufWkt::Struct& value) { + stream_info_->setDynamicMetadata(name, value); +} + +void ActiveStreamSocket::newConnection() { + connected_ = true; + + // Check if the socket may need to be redirected to another listener. + Network::BalancedConnectionHandlerOptRef new_listener; + + if (hand_off_restored_destination_connections_ && + socket_->addressProvider().localAddressRestored()) { + // Find a listener associated with the original destination address. + new_listener = + listener_.getBalancedHandlerByAddress(*socket_->addressProvider().localAddress()); + } + if (new_listener.has_value()) { + // Hands off connections redirected by iptables to the listener associated with the + // original destination address. Pass 'hand_off_restored_destination_connections' as false to + // prevent further redirection. + // Leave the new listener to decide whether to execute re-balance. + // Note also that we must account for the number of connections properly across both listeners. + // TODO(mattklein123): See note in ~ActiveTcpSocket() related to making this accounting better. + listener_.decNumConnections(); + new_listener.value().get().onAcceptWorker(std::move(socket_), false, false); + } else { + // Set default transport protocol if none of the listener filters did it. + if (socket_->detectedTransportProtocol().empty()) { + socket_->setDetectedTransportProtocol("raw_buffer"); + } + // TODO(lambdai): add integration test + // TODO: Address issues in wider scope. See https://github.com/envoyproxy/envoy/issues/8925 + // Erase accept filter states because accept filters may not get the opportunity to clean up. + // Particularly the assigned events need to reset before assigning new events in the follow up. + accept_filters_.clear(); + // Create a new connection on this listener. + listener_.newConnection(std::move(socket_), std::move(stream_info_)); + } +} +} // namespace Server +} // namespace Envoy diff --git a/source/server/active_stream_socket.h b/source/server/active_stream_socket.h new file mode 100644 index 0000000000000..efe2fa98bbb7b --- /dev/null +++ b/source/server/active_stream_socket.h @@ -0,0 +1,224 @@ +#pragma once + +#pragma once +#include +#include +#include +#include + +#include "envoy/common/time.h" +#include "envoy/event/deferred_deletable.h" +#include "envoy/event/dispatcher.h" +#include "envoy/network/connection.h" +#include "envoy/network/connection_handler.h" +#include "envoy/network/filter.h" +#include "envoy/network/listen_socket.h" +#include "envoy/network/listener.h" +#include "envoy/server/listener_manager.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/timespan.h" + +#include "common/common/linked_object.h" +#include "common/common/non_copyable.h" +#include "common/stream_info/stream_info_impl.h" + +#include "server/active_listener_base.h" + +#include "spdlog/spdlog.h" + +namespace Envoy { +namespace Server { + +class ActiveStreamListenerBase; + +/** + * Wrapper for an active accepted socket owned by the active listener. + */ +struct ActiveStreamSocket : public Network::ListenerFilterManager, + public Network::ListenerFilterCallbacks, + LinkedObject, + public Event::DeferredDeletable, + Logger::Loggable { + ActiveStreamSocket(ActiveStreamListenerBase& listener, Network::ConnectionSocketPtr&& socket, + bool hand_off_restored_destination_connections); + ~ActiveStreamSocket() override; + + void onTimeout(); + void startTimer(); + void unlink(); + void newConnection(); + + class GenericListenerFilter : public Network::ListenerFilter { + public: + GenericListenerFilter(const Network::ListenerFilterMatcherSharedPtr& matcher, + Network::ListenerFilterPtr listener_filter) + : listener_filter_(std::move(listener_filter)), matcher_(std::move(matcher)) {} + Network::FilterStatus onAccept(ListenerFilterCallbacks& cb) override { + if (isDisabled(cb)) { + return Network::FilterStatus::Continue; + } + return listener_filter_->onAccept(cb); + } + /** + * Check if this filter filter should be disabled on the incoming socket. + * @param cb the callbacks the filter instance can use to communicate with the filter chain. + **/ + bool isDisabled(ListenerFilterCallbacks& cb) { + if (matcher_ == nullptr) { + return false; + } else { + return matcher_->matches(cb); + } + } + + private: + const Network::ListenerFilterPtr listener_filter_; + const Network::ListenerFilterMatcherSharedPtr matcher_; + }; + using ListenerFilterWrapperPtr = std::unique_ptr; + + // Network::ListenerFilterManager + void addAcceptFilter(const Network::ListenerFilterMatcherSharedPtr& listener_filter_matcher, + Network::ListenerFilterPtr&& filter) override { + accept_filters_.emplace_back( + std::make_unique(listener_filter_matcher, std::move(filter))); + } + + // Network::ListenerFilterCallbacks + Network::ConnectionSocket& socket() override { return *socket_.get(); } + Event::Dispatcher& dispatcher() override; + void continueFilterChain(bool success) override; + void setDynamicMetadata(const std::string& name, const ProtobufWkt::Struct& value) override; + envoy::config::core::v3::Metadata& dynamicMetadata() override { + return stream_info_->dynamicMetadata(); + }; + const envoy::config::core::v3::Metadata& dynamicMetadata() const override { + return stream_info_->dynamicMetadata(); + }; + + StreamInfo::FilterState& filterState() override { return *stream_info_->filterState().get(); } + + ActiveStreamListenerBase& listener_; + Network::ConnectionSocketPtr socket_; + const bool hand_off_restored_destination_connections_; + std::list accept_filters_; + std::list::iterator iter_; + Event::TimerPtr timer_; + std::unique_ptr stream_info_; + bool connected_{false}; +}; + +using ActiveInternalSocket = ActiveStreamSocket; + +class ActiveStreamListenerBase : public ActiveListenerImplBase { +public: + ActiveStreamListenerBase(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, + Network::ListenerPtr&& listener, Network::ListenerConfig& config); + static void emitLogs(Network::ListenerConfig& config, StreamInfo::StreamInfo& stream_info); + + virtual void incNumConnections() PURE; + virtual void decNumConnections() PURE; + virtual Event::Dispatcher& dispatcher() PURE; + + /** + * Create a new connection from a socket accepted by the listener. + */ + virtual void newConnection(Network::ConnectionSocketPtr&& socket, + std::unique_ptr stream_info) PURE; + + virtual Network::BalancedConnectionHandlerOptRef + getBalancedHandlerByAddress(const Network::Address::Instance& address) PURE; + + void onSocketAccepted(std::unique_ptr active_socket) { + // Create and run the filters + config_->filterChainFactory().createListenerFilterChain(*active_socket); + active_socket->continueFilterChain(true); + + // Move active_socket to the sockets_ list if filter iteration needs to continue later. + // Otherwise we let active_socket be destructed when it goes out of scope. + if (active_socket->iter_ != active_socket->accept_filters_.end()) { + active_socket->startTimer(); + LinkedList::moveIntoListBack(std::move(active_socket), sockets_); + } else { + // If active_socket is about to be destructed, emit logs if a connection is not created. + if (!active_socket->connected_) { + emitLogs(*config_, *active_socket->stream_info_); + } + } + } + + Network::ConnectionHandler& parent_; + Event::Dispatcher& dispatcher_; + Network::ListenerPtr listener_; + const std::chrono::milliseconds listener_filters_timeout_; + const bool continue_on_listener_filters_timeout_; + std::list> sockets_; +}; + +template +class TypedActiveStreamListenerBase : public ActiveStreamListenerBase { + +public: + TypedActiveStreamListenerBase(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, + Network::ListenerPtr&& listener, Network::ListenerConfig& config) + : ActiveStreamListenerBase(parent, dispatcher, std::move(listener), config) {} + using ActiveGenericConnectionPtr = std::unique_ptr; + using ActiveGenericConnectionsPtr = std::unique_ptr; + + /** + * Schedule to remove and destroy the active connections which are not tracked by listener + * config. Caution: The connection are not destroyed yet when function returns. + */ + void + deferredRemoveFilterChains(const std::list& draining_filter_chains) { + // Need to recover the original deleting state. + const bool was_deleting = is_deleting_; + is_deleting_ = true; + for (const auto* filter_chain : draining_filter_chains) { + auto iter = connections_by_context_.find(filter_chain); + if (iter == connections_by_context_.end()) { + // It is possible when listener is stopping. + } else { + auto& connections = iter->second->connections_; + while (!connections.empty()) { + connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); + } + // Since is_deleting_ is on, we need to manually remove the map value and drive the + // iterator. Defer delete connection container to avoid race condition in destroying + // connection. + dispatcher().deferredDelete(std::move(iter->second)); + connections_by_context_.erase(iter); + } + } + is_deleting_ = was_deleting; + } + +protected: + void cleanupConnections() { + is_deleting_ = true; + + // Purge sockets that have not progressed to connections. This should only happen when + // a listener filter stops iteration and never resumes. + while (!sockets_.empty()) { + auto removed = sockets_.front()->removeFromList(sockets_); + dispatcher().deferredDelete(std::move(removed)); + } + + for (auto& chain_and_connections : connections_by_context_) { + ASSERT(chain_and_connections.second != nullptr); + auto& connections = chain_and_connections.second->connections_; + while (!connections.empty()) { + connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); + } + } + dispatcher().clearDeferredDeleteList(); + } + + absl::node_hash_map + connections_by_context_; + bool is_deleting_{false}; +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/active_tcp_listener.cc b/source/server/active_tcp_listener.cc index bcf5cae34d836..54d7a82e9892e 100644 --- a/source/server/active_tcp_listener.cc +++ b/source/server/active_tcp_listener.cc @@ -15,51 +15,30 @@ namespace Envoy { namespace Server { -namespace { -void emitLogs(Network::ListenerConfig& config, StreamInfo::StreamInfo& stream_info) { - stream_info.onRequestComplete(); - for (const auto& access_log : config.accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, stream_info); - } -} -} // namespace - ActiveTcpListener::ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerConfig& config) - : ActiveTcpListener( - parent, + : TypedActiveStreamListenerBase( + parent, parent.dispatcher(), parent.dispatcher().createListener(config.listenSocketFactory().getListenSocket(), *this, config.bindToPort(), config.tcpBacklogSize()), - config) {} + config), + tcp_conn_handler_(parent) { + config.connectionBalancer().registerHandler(*this); +} ActiveTcpListener::ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerPtr&& listener, Network::ListenerConfig& config) - : ActiveListenerImplBase(parent, &config), parent_(parent), listener_(std::move(listener)), - listener_filters_timeout_(config.listenerFiltersTimeout()), - continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()) { + : TypedActiveStreamListenerBase(parent, parent.dispatcher(), + std::move(listener), config), + tcp_conn_handler_(parent) { config.connectionBalancer().registerHandler(*this); } ActiveTcpListener::~ActiveTcpListener() { - is_deleting_ = true; config_->connectionBalancer().unregisterHandler(*this); - // Purge sockets that have not progressed to connections. This should only happen when - // a listener filter stops iteration and never resumes. - while (!sockets_.empty()) { - ActiveTcpSocketPtr removed = sockets_.front()->removeFromList(sockets_); - parent_.dispatcher().deferredDelete(std::move(removed)); - } - - for (auto& chain_and_connections : connections_by_context_) { - ASSERT(chain_and_connections.second != nullptr); - auto& connections = chain_and_connections.second->connections_; - while (!connections.empty()) { - connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); - } - } - parent_.dispatcher().clearDeferredDeleteList(); + cleanupConnections(); // By the time a listener is destroyed, in the common case, there should be no connections. // However, this is not always true if there is an in flight rebalanced connection that is @@ -75,14 +54,14 @@ void ActiveTcpListener::removeConnection(ActiveTcpConnection& connection) { ENVOY_CONN_LOG(debug, "adding to cleanup list", *connection.connection_); ActiveConnections& active_connections = connection.active_connections_; ActiveTcpConnectionPtr removed = connection.removeFromList(active_connections.connections_); - parent_.dispatcher().deferredDelete(std::move(removed)); + dispatcher().deferredDelete(std::move(removed)); // Delete map entry only iff connections becomes empty. if (active_connections.connections_.empty()) { auto iter = connections_by_context_.find(&active_connections.filter_chain_); ASSERT(iter != connections_by_context_.end()); // To cover the lifetime of every single connection, Connections need to be deferred deleted // because the previously contained connection is deferred deleted. - parent_.dispatcher().deferredDelete(std::move(iter->second)); + dispatcher().deferredDelete(std::move(iter->second)); // The erase will break the iteration over the connections_by_context_ during the deletion. if (!is_deleting_) { connections_by_context_.erase(iter); @@ -96,118 +75,6 @@ void ActiveTcpListener::updateListenerConfig(Network::ListenerConfig& config) { config_ = &config; } -void ActiveTcpSocket::onTimeout() { - listener_.stats_.downstream_pre_cx_timeout_.inc(); - ASSERT(inserted()); - ENVOY_LOG(debug, "listener filter times out after {} ms", - listener_.listener_filters_timeout_.count()); - - if (listener_.continue_on_listener_filters_timeout_) { - ENVOY_LOG(debug, "fallback to default listener filter"); - newConnection(); - } - unlink(); -} - -void ActiveTcpSocket::startTimer() { - if (listener_.listener_filters_timeout_.count() > 0) { - timer_ = listener_.parent_.dispatcher().createTimer([this]() -> void { onTimeout(); }); - timer_->enableTimer(listener_.listener_filters_timeout_); - } -} - -void ActiveTcpSocket::unlink() { - ActiveTcpSocketPtr removed = removeFromList(listener_.sockets_); - if (removed->timer_ != nullptr) { - removed->timer_->disableTimer(); - } - // Emit logs if a connection is not established. - if (!connected_) { - emitLogs(*listener_.config_, *stream_info_); - } - listener_.parent_.dispatcher().deferredDelete(std::move(removed)); -} - -void ActiveTcpSocket::continueFilterChain(bool success) { - if (success) { - bool no_error = true; - if (iter_ == accept_filters_.end()) { - iter_ = accept_filters_.begin(); - } else { - iter_ = std::next(iter_); - } - - for (; iter_ != accept_filters_.end(); iter_++) { - Network::FilterStatus status = (*iter_)->onAccept(*this); - if (status == Network::FilterStatus::StopIteration) { - // The filter is responsible for calling us again at a later time to continue the filter - // chain from the next filter. - if (!socket().ioHandle().isOpen()) { - // break the loop but should not create new connection - no_error = false; - break; - } else { - // Blocking at the filter but no error - return; - } - } - } - // Successfully ran all the accept filters. - if (no_error) { - newConnection(); - } else { - // Signal the caller that no extra filter chain iteration is needed. - iter_ = accept_filters_.end(); - } - } - - // Filter execution concluded, unlink and delete this ActiveTcpSocket if it was linked. - if (inserted()) { - unlink(); - } -} - -void ActiveTcpSocket::setDynamicMetadata(const std::string& name, - const ProtobufWkt::Struct& value) { - stream_info_->setDynamicMetadata(name, value); -} - -void ActiveTcpSocket::newConnection() { - connected_ = true; - - // Check if the socket may need to be redirected to another listener. - Network::BalancedConnectionHandlerOptRef new_listener; - - if (hand_off_restored_destination_connections_ && - socket_->addressProvider().localAddressRestored()) { - // Find a listener associated with the original destination address. - new_listener = - listener_.parent_.getBalancedHandlerByAddress(*socket_->addressProvider().localAddress()); - } - if (new_listener.has_value()) { - // Hands off connections redirected by iptables to the listener associated with the - // original destination address. Pass 'hand_off_restored_destination_connections' as false to - // prevent further redirection. - // Leave the new listener to decide whether to execute re-balance. - // Note also that we must account for the number of connections properly across both listeners. - // TODO(mattklein123): See note in ~ActiveTcpSocket() related to making this accounting better. - listener_.decNumConnections(); - new_listener.value().get().onAcceptWorker(std::move(socket_), false, false); - } else { - // Set default transport protocol if none of the listener filters did it. - if (socket_->detectedTransportProtocol().empty()) { - socket_->setDetectedTransportProtocol("raw_buffer"); - } - // TODO(lambdai): add integration test - // TODO: Address issues in wider scope. See https://github.com/envoyproxy/envoy/issues/8925 - // Erase accept filter states because accept filters may not get the opportunity to clean up. - // Particularly the assigned events need to reset before assigning new events in the follow up. - accept_filters_.clear(); - // Create a new connection on this listener. - listener_.newConnection(std::move(socket_), std::move(stream_info_)); - } -} - void ActiveTcpListener::onAccept(Network::ConnectionSocketPtr&& socket) { if (listenerConnectionLimitReached()) { ENVOY_LOG(trace, "closing connection: listener connection limit reached for {}", @@ -243,31 +110,15 @@ void ActiveTcpListener::onAcceptWorker(Network::ConnectionSocketPtr&& socket, } } - auto active_socket = std::make_unique(*this, std::move(socket), - hand_off_restored_destination_connections); - - // Create and run the filters. - config_->filterChainFactory().createListenerFilterChain(*active_socket); - active_socket->continueFilterChain(true); - - // Move active_socket to the sockets_ list if filter iteration needs to continue later. - // Otherwise we let active_socket be destructed when it goes out of scope. - if (active_socket->iter_ != active_socket->accept_filters_.end()) { - active_socket->startTimer(); - LinkedList::moveIntoListBack(std::move(active_socket), sockets_); - } else { - // If active_socket is about to be destructed, emit logs if a connection is not created. - if (!active_socket->connected_) { - if (active_socket->stream_info_ != nullptr) { - emitLogs(*config_, *active_socket->stream_info_); - } else { - // If the active_socket is not connected, this socket is not promoted to active connection. - // Thus the stream_info_ is owned by this active socket. - ENVOY_BUG(active_socket->stream_info_ != nullptr, - "the unconnected active socket must have stream info."); - } - } - } + auto active_socket = std::make_unique( + *this, std::move(socket), hand_off_restored_destination_connections); + + onSocketAccepted(std::move(active_socket)); +} + +Network::BalancedConnectionHandlerOptRef +ActiveTcpListener::getBalancedHandlerByAddress(const Network::Address::Instance& address) { + return tcp_conn_handler_.getBalancedHandlerByAddress(address); } void ActiveTcpListener::pauseListening() { @@ -301,7 +152,7 @@ void ActiveTcpListener::newConnection(Network::ConnectionSocketPtr&& socket, auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); stream_info->setDownstreamSslConnection(transport_socket->ssl()); auto& active_connections = getOrCreateActiveConnections(*filter_chain); - auto server_conn_ptr = parent_.dispatcher().createServerConnection( + auto server_conn_ptr = dispatcher().createServerConnection( std::move(socket), std::move(transport_socket), *stream_info); if (const auto timeout = filter_chain->transportSocketConnectTimeout(); timeout != std::chrono::milliseconds::zero()) { @@ -309,7 +160,7 @@ void ActiveTcpListener::newConnection(Network::ConnectionSocketPtr&& socket, } ActiveTcpConnectionPtr active_connection( new ActiveTcpConnection(active_connections, std::move(server_conn_ptr), - parent_.dispatcher().timeSource(), std::move(stream_info))); + dispatcher().timeSource(), std::move(stream_info))); active_connection->connection_->setBufferLimits(config_->perConnectionBufferLimitBytes()); const bool empty_filter_chain = !config_->filterChainFactory().createNetworkFilterChain( @@ -336,29 +187,6 @@ ActiveTcpListener::getOrCreateActiveConnections(const Network::FilterChain& filt return *connections; } -void ActiveTcpListener::deferredRemoveFilterChains( - const std::list& draining_filter_chains) { - // Need to recover the original deleting state. - const bool was_deleting = is_deleting_; - is_deleting_ = true; - for (const auto* filter_chain : draining_filter_chains) { - auto iter = connections_by_context_.find(filter_chain); - if (iter == connections_by_context_.end()) { - // It is possible when listener is stopping. - } else { - auto& connections = iter->second->connections_; - while (!connections.empty()) { - connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); - } - // Since is_deleting_ is on, we need to manually remove the map value and drive the iterator. - // Defer delete connection container to avoid race condition in destroying connection. - parent_.dispatcher().deferredDelete(std::move(iter->second)); - connections_by_context_.erase(iter); - } - } - is_deleting_ = was_deleting; -} - void ActiveTcpListener::post(Network::ConnectionSocketPtr&& socket) { // It is not possible to capture a unique_ptr because the post() API copies the lambda, so we must // bundle the socket inside a shared_ptr that can be captured. @@ -367,9 +195,10 @@ void ActiveTcpListener::post(Network::ConnectionSocketPtr&& socket) { RebalancedSocketSharedPtr socket_to_rebalance = std::make_shared(); socket_to_rebalance->socket = std::move(socket); - parent_.dispatcher().post([socket_to_rebalance, tag = config_->listenerTag(), &parent = parent_, - handoff = config_->handOffRestoredDestinationConnections()]() { - auto balanced_handler = parent.getBalancedHandlerByTag(tag); + dispatcher().post([socket_to_rebalance, tag = config_->listenerTag(), + &tcp_conn_handler = tcp_conn_handler_, + handoff = config_->handOffRestoredDestinationConnections()]() { + auto balanced_handler = tcp_conn_handler.getBalancedHandlerByTag(tag); if (balanced_handler.has_value()) { balanced_handler->get().onAcceptWorker(std::move(socket_to_rebalance->socket), handoff, true); return; @@ -411,7 +240,7 @@ ActiveTcpConnection::ActiveTcpConnection(ActiveConnections& active_connections, } ActiveTcpConnection::~ActiveTcpConnection() { - emitLogs(*active_connections_.listener_.config_, *stream_info_); + ActiveStreamListenerBase::emitLogs(*active_connections_.listener_.config_, *stream_info_); auto& listener = active_connections_.listener_; listener.stats_.downstream_cx_active_.dec(); listener.stats_.downstream_cx_destroy_.inc(); @@ -425,5 +254,16 @@ ActiveTcpConnection::~ActiveTcpConnection() { listener.parent_.decNumConnections(); } +// Network::ConnectionCallbacks +void ActiveTcpConnection::onEvent(Network::ConnectionEvent event) { + FANCY_LOG(info, "lambdai: tcp connection on event {}", static_cast(event)); + + // Any event leads to destruction of the connection. + if (event == Network::ConnectionEvent::LocalClose || + event == Network::ConnectionEvent::RemoteClose) { + active_connections_.listener_.removeConnection(*this); + } +} + } // namespace Server } // namespace Envoy diff --git a/source/server/active_tcp_listener.h b/source/server/active_tcp_listener.h index c698faaa06058..09a282f077a40 100644 --- a/source/server/active_tcp_listener.h +++ b/source/server/active_tcp_listener.h @@ -8,17 +8,18 @@ #include "common/stream_info/stream_info_impl.h" #include "server/active_listener_base.h" +#include "server/active_stream_socket.h" namespace Envoy { namespace Server { struct ActiveTcpConnection; using ActiveTcpConnectionPtr = std::unique_ptr; -struct ActiveTcpSocket; -using ActiveTcpSocketPtr = std::unique_ptr; class ActiveConnections; using ActiveConnectionsPtr = std::unique_ptr; +class ActiveTcpListener; + namespace { // Structure used to allow a unique_ptr to be captured in a posted lambda. See below. struct RebalancedSocket { @@ -27,11 +28,48 @@ struct RebalancedSocket { using RebalancedSocketSharedPtr = std::shared_ptr; } // namespace +/** + * Wrapper for a group of active connections which are attached to the same filter chain context. + */ +class ActiveConnections : public Event::DeferredDeletable { +public: + ActiveConnections(ActiveTcpListener& listener, const Network::FilterChain& filter_chain); + ~ActiveConnections() override; + + // listener filter chain pair is the owner of the connections + ActiveTcpListener& listener_; + const Network::FilterChain& filter_chain_; + // Owned connections + std::list connections_; +}; + +/** + * Wrapper for an active TCP connection owned by this handler. + */ +struct ActiveTcpConnection : LinkedObject, + public Event::DeferredDeletable, + public Network::ConnectionCallbacks { + ActiveTcpConnection(ActiveConnections& active_connections, + Network::ConnectionPtr&& new_connection, TimeSource& time_system, + std::unique_ptr&& stream_info); + ~ActiveTcpConnection() override; + using CollectionType = ActiveConnections; + // Network::ConnectionCallbacks + void onEvent(Network::ConnectionEvent event) override; + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} + + std::unique_ptr stream_info_; + ActiveConnections& active_connections_; + Network::ConnectionPtr connection_; + Stats::TimespanPtr conn_length_; +}; + /** * Wrapper for an active tcp listener owned by this handler. */ class ActiveTcpListener final : public Network::TcpListenerCallbacks, - public ActiveListenerImplBase, + public TypedActiveStreamListenerBase, public Network::BalancedConnectionHandler, Logger::Loggable { public: @@ -45,7 +83,7 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, return !config_->openConnections().canCreate(); } - void decNumConnections() { + void decNumConnections() override { ASSERT(num_listener_connections_ > 0); --num_listener_connections_; config_->openConnections().dec(); @@ -57,6 +95,10 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, // ActiveListenerImplBase Network::Listener* listener() override { return listener_.get(); } + Event::Dispatcher& dispatcher() override { return dispatcher_; } + Network::BalancedConnectionHandlerOptRef + getBalancedHandlerByAddress(const Network::Address::Instance& address) override; + void pauseListening() override; void resumeListening() override; void shutdownListener() override { listener_.reset(); } @@ -81,180 +123,25 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, * Create a new connection from a socket accepted by the listener. */ void newConnection(Network::ConnectionSocketPtr&& socket, - std::unique_ptr stream_info); + std::unique_ptr stream_info) override; /** * Return the active connections container attached with the given filter chain. */ ActiveConnections& getOrCreateActiveConnections(const Network::FilterChain& filter_chain); - /** - * Schedule to remove and destroy the active connections which are not tracked by listener - * config. Caution: The connection are not destroyed yet when function returns. - */ - void - deferredRemoveFilterChains(const std::list& draining_filter_chains); - /** * Update the listener config. The follow up connections will see the new config. The existing * connections are not impacted. */ void updateListenerConfig(Network::ListenerConfig& config); - Network::TcpConnectionHandler& parent_; - Network::ListenerPtr listener_; - const std::chrono::milliseconds listener_filters_timeout_; - const bool continue_on_listener_filters_timeout_; - std::list sockets_; - absl::flat_hash_map connections_by_context_; - + Network::TcpConnectionHandler& tcp_conn_handler_; // The number of connections currently active on this listener. This is typically used for // connection balancing across per-handler listeners. std::atomic num_listener_connections_{}; - bool is_deleting_{false}; -}; - -/** - * Wrapper for a group of active connections which are attached to the same filter chain context. - */ -class ActiveConnections : public Event::DeferredDeletable { -public: - ActiveConnections(ActiveTcpListener& listener, const Network::FilterChain& filter_chain); - ~ActiveConnections() override; - - // listener filter chain pair is the owner of the connections - ActiveTcpListener& listener_; - const Network::FilterChain& filter_chain_; - // Owned connections - std::list connections_; }; -/** - * Wrapper for an active TCP connection owned by this handler. - */ -struct ActiveTcpConnection : LinkedObject, - public Event::DeferredDeletable, - public Network::ConnectionCallbacks { - ActiveTcpConnection(ActiveConnections& active_connections, - Network::ConnectionPtr&& new_connection, TimeSource& time_system, - std::unique_ptr&& stream_info); - ~ActiveTcpConnection() override; - - // Network::ConnectionCallbacks - void onEvent(Network::ConnectionEvent event) override { - // Any event leads to destruction of the connection. - if (event == Network::ConnectionEvent::LocalClose || - event == Network::ConnectionEvent::RemoteClose) { - active_connections_.listener_.removeConnection(*this); - } - } - void onAboveWriteBufferHighWatermark() override {} - void onBelowWriteBufferLowWatermark() override {} - - std::unique_ptr stream_info_; - ActiveConnections& active_connections_; - Network::ConnectionPtr connection_; - Stats::TimespanPtr conn_length_; -}; - -/** - * Wrapper for an active accepted TCP socket owned by this handler. - */ -struct ActiveTcpSocket : public Network::ListenerFilterManager, - public Network::ListenerFilterCallbacks, - LinkedObject, - public Event::DeferredDeletable, - Logger::Loggable { - ActiveTcpSocket(ActiveTcpListener& listener, Network::ConnectionSocketPtr&& socket, - bool hand_off_restored_destination_connections) - : listener_(listener), socket_(std::move(socket)), - hand_off_restored_destination_connections_(hand_off_restored_destination_connections), - iter_(accept_filters_.end()), - stream_info_(std::make_unique( - listener_.parent_.dispatcher().timeSource(), socket_->addressProviderSharedPtr(), - StreamInfo::FilterState::LifeSpan::Connection)) { - listener_.stats_.downstream_pre_cx_active_.inc(); - } - ~ActiveTcpSocket() override { - accept_filters_.clear(); - listener_.stats_.downstream_pre_cx_active_.dec(); - - // If the underlying socket is no longer attached, it means that it has been transferred to - // an active connection. In this case, the active connection will decrement the number - // of listener connections. - // TODO(mattklein123): In general the way we account for the number of listener connections - // is incredibly fragile. Revisit this by potentially merging ActiveTcpSocket and - // ActiveTcpConnection, having a shared object which does accounting (but would require - // another allocation, etc.). - if (socket_ != nullptr) { - listener_.decNumConnections(); - } - } - - void onTimeout(); - void startTimer(); - void unlink(); - void newConnection(); - - class GenericListenerFilter : public Network::ListenerFilter { - public: - GenericListenerFilter(const Network::ListenerFilterMatcherSharedPtr& matcher, - Network::ListenerFilterPtr listener_filter) - : listener_filter_(std::move(listener_filter)), matcher_(std::move(matcher)) {} - Network::FilterStatus onAccept(ListenerFilterCallbacks& cb) override { - if (isDisabled(cb)) { - return Network::FilterStatus::Continue; - } - return listener_filter_->onAccept(cb); - } - /** - * Check if this filter filter should be disabled on the incoming socket. - * @param cb the callbacks the filter instance can use to communicate with the filter chain. - **/ - bool isDisabled(ListenerFilterCallbacks& cb) { - if (matcher_ == nullptr) { - return false; - } else { - return matcher_->matches(cb); - } - } - - private: - const Network::ListenerFilterPtr listener_filter_; - const Network::ListenerFilterMatcherSharedPtr matcher_; - }; - using ListenerFilterWrapperPtr = std::unique_ptr; - - // Network::ListenerFilterManager - void addAcceptFilter(const Network::ListenerFilterMatcherSharedPtr& listener_filter_matcher, - Network::ListenerFilterPtr&& filter) override { - accept_filters_.emplace_back( - std::make_unique(listener_filter_matcher, std::move(filter))); - } - - // Network::ListenerFilterCallbacks - Network::ConnectionSocket& socket() override { return *socket_.get(); } - Event::Dispatcher& dispatcher() override { return listener_.parent_.dispatcher(); } - void continueFilterChain(bool success) override; - void setDynamicMetadata(const std::string& name, const ProtobufWkt::Struct& value) override; - envoy::config::core::v3::Metadata& dynamicMetadata() override { - return stream_info_->dynamicMetadata(); - }; - const envoy::config::core::v3::Metadata& dynamicMetadata() const override { - return stream_info_->dynamicMetadata(); - }; - - StreamInfo::FilterState& filterState() override { return *stream_info_->filterState().get(); } - - ActiveTcpListener& listener_; - Network::ConnectionSocketPtr socket_; - const bool hand_off_restored_destination_connections_; - std::list accept_filters_; - std::list::iterator iter_; - Event::TimerPtr timer_; - std::unique_ptr stream_info_; - bool connected_{false}; -}; using ActiveTcpListenerOptRef = absl::optional>; } // namespace Server diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index 84d5236e212bc..78345c70b7ef3 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -360,6 +360,9 @@ class AdminImpl : public Admin, Network::UdpListenerConfigOptRef udpListenerConfig() override { return Network::UdpListenerConfigOptRef(); } + Network::InternalListenerConfigOptRef internalListenerConfig() override { + return Network::InternalListenerConfigOptRef(); + } envoy::config::core::v3::TrafficDirection direction() const override { return envoy::config::core::v3::UNSPECIFIED; } diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index 84ea2bbc416fc..d843d1929f4fb 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -5,9 +5,11 @@ #include "envoy/event/dispatcher.h" #include "envoy/network/filter.h" +#include "common/common/logger.h" #include "common/event/deferred_task.h" #include "common/network/utility.h" +#include "server/active_internal_listener.h" #include "server/active_tcp_listener.h" namespace Envoy { @@ -28,7 +30,20 @@ void ConnectionHandlerImpl::decNumConnections() { void ConnectionHandlerImpl::addListener(absl::optional overridden_listener, Network::ListenerConfig& config) { ActiveListenerDetails details; - if (config.listenSocketFactory().socketType() == Network::Socket::Type::Stream) { + if (config.internalListenerConfig().has_value()) { + if (overridden_listener.has_value()) { + for (auto& listener : listeners_) { + if (listener.second.listener_->listenerTag() == overridden_listener) { + listener.second.internalListener()->get().updateListenerConfig(config); + return; + } + } + NOT_REACHED_GCOVR_EXCL_LINE; + } + auto internal_listener = std::make_unique(*this, dispatcher(), config); + details.typed_listener_ = *internal_listener; + details.listener_ = std::move(internal_listener); + } else if (config.listenSocketFactory().socketType() == Network::Socket::Type::Stream) { if (overridden_listener.has_value()) { for (auto& listener : listeners_) { if (listener.second.listener_->listenerTag() == overridden_listener) { @@ -132,7 +147,27 @@ void ConnectionHandlerImpl::setListenerRejectFraction(UnitFloat reject_fraction) } } -ActiveTcpListenerOptRef ConnectionHandlerImpl::ActiveListenerDetails::tcpListener() { +Network::InternalListenerCallbacksOptRef +ConnectionHandlerImpl::findByAddress(const Network::Address::InstanceConstSharedPtr& address) { + auto listener_it = + std::find_if(listeners_.begin(), listeners_.end(), + [&address](std::pair& p) { + return p.second.internalListener().has_value() && + p.second.listener_->listener() != nullptr && + p.first->type() == Network::Address::Type::EnvoyInternal && + *(p.first) == *address; + }); + + if (listener_it != listeners_.end()) { + return Network::InternalListenerCallbacksOptRef( + listener_it->second.internalListener().value().get()); + } + return absl::nullopt; +} + +ConnectionHandlerImpl::ActiveTcpListenerOptRef +ConnectionHandlerImpl::ActiveListenerDetails::tcpListener() { auto* val = absl::get_if>(&typed_listener_); return (val != nullptr) ? absl::make_optional(*val) : absl::nullopt; } @@ -143,6 +178,12 @@ ConnectionHandlerImpl::ActiveListenerDetails::udpListener() { return (val != nullptr) ? absl::make_optional(*val) : absl::nullopt; } +ConnectionHandlerImpl::ActiveInternalListenerOptRef +ConnectionHandlerImpl::ActiveListenerDetails::internalListener() { + auto* val = absl::get_if>(&typed_listener_); + return (val != nullptr) ? absl::make_optional(*val) : absl::nullopt; +} + ConnectionHandlerImpl::ActiveListenerDetailsOptRef ConnectionHandlerImpl::findActiveListenerByTag(uint64_t listener_tag) { // TODO(mattklein123): We should probably use a hash table here to lookup the tag diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index 4d73ba79fe219..911a3d32084c5 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -21,6 +21,7 @@ namespace Server { class ActiveUdpListenerBase; class ActiveTcpListener; +class ActiveInternalListener; /** * Server side connection handler. This is used both by workers as well as the @@ -28,12 +29,15 @@ class ActiveTcpListener; */ class ConnectionHandlerImpl : public Network::TcpConnectionHandler, public Network::UdpConnectionHandler, + public Network::InternalListenerManager, NonCopyable, Logger::Loggable { public: using UdpListenerCallbacksOptRef = absl::optional>; using ActiveTcpListenerOptRef = absl::optional>; + using ActiveInternalListenerOptRef = + absl::optional>; ConnectionHandlerImpl(Event::Dispatcher& dispatcher, absl::optional worker_index); @@ -63,20 +67,24 @@ class ConnectionHandlerImpl : public Network::TcpConnectionHandler, // Network::UdpConnectionHandler Network::UdpListenerCallbacksOptRef getUdpListenerCallbacks(uint64_t listener_tag) override; + // Network::InternalListenerManager + Network::InternalListenerCallbacksOptRef + findByAddress(const Network::Address::InstanceConstSharedPtr& listen_address) override; + private: struct ActiveListenerDetails { // Strong pointer to the listener, whether TCP, UDP, QUIC, etc. Network::ConnectionHandler::ActiveListenerPtr listener_; - absl::variant, - std::reference_wrapper> + absl::variant, + std::reference_wrapper, + std::reference_wrapper> typed_listener_; // Helpers for accessing the data in the variant for cleaner code. ActiveTcpListenerOptRef tcpListener(); UdpListenerCallbacksOptRef udpListener(); + ActiveInternalListenerOptRef internalListener(); }; using ActiveListenerDetailsOptRef = absl::optional>; ActiveListenerDetailsOptRef findActiveListenerByTag(uint64_t listener_tag); diff --git a/source/server/listener_impl.cc b/source/server/listener_impl.cc index 405937f7cf7b1..35541132b89d1 100644 --- a/source/server/listener_impl.cc +++ b/source/server/listener_impl.cc @@ -1,5 +1,7 @@ #include "server/listener_impl.h" +#include + #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/listener/v3/listener.pb.h" #include "envoy/config/listener/v3/listener_components.pb.h" @@ -94,7 +96,8 @@ ListenSocketFactoryImpl::ListenSocketFactoryImpl(ListenerComponentFactory& facto bind_to_port_(bind_to_port), listener_name_(listener_name), reuse_port_(reuse_port) { bool create_socket = false; - if (local_address_->type() == Network::Address::Type::Ip) { + switch (local_address_->type()) { + case Network::Address::Type::Ip: if (socket_type_ == Network::Socket::Type::Datagram) { ASSERT(reuse_port_ == true); } @@ -107,10 +110,14 @@ ListenSocketFactoryImpl::ListenSocketFactoryImpl(ListenerComponentFactory& facto // then all worker threads should use same port. create_socket = true; } - } else { - ASSERT(local_address_->type() == Network::Address::Type::Pipe); + break; + case Network::Address::Type::Pipe: // Listeners with Unix domain socket always use shared socket. create_socket = true; + break; + case Network::Address::Type::EnvoyInternal: + create_socket = false; + break; } if (create_socket) { @@ -320,6 +327,7 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, buildOriginalDstListenerFilter(); buildProxyProtocolListenerFilter(); buildTlsInspectorListenerFilter(); + buildInternalListener(); } if (!workers_started_) { // Initialize dynamic_init_manager_ from Server's init manager if it's not initialized. @@ -347,7 +355,8 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, validation_visitor_( added_via_api_ ? parent_.server_.messageValidationContext().dynamicValidationVisitor() : parent_.server_.messageValidationContext().staticValidationVisitor()), - // listener_init_target_ is not used during in place update because we expect server started. + // listener_init_target_ is not used during in place update because we expect server + // started. listener_init_target_("", nullptr), dynamic_init_manager_(std::make_unique( fmt::format("Listener-local-init-manager {} {}", name, hash))), @@ -376,6 +385,7 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, buildOriginalDstListenerFilter(); buildProxyProtocolListenerFilter(); buildTlsInspectorListenerFilter(); + buildInternalListener(); open_connections_ = origin.open_connections_; } @@ -387,6 +397,12 @@ void ListenerImpl::buildAccessLog() { } } +void ListenerImpl::buildInternalListener() { + if (config_.has_internal_listener()) { + internal_listener_config_ = std::make_unique(); + } +} + void ListenerImpl::buildUdpListenerFactory(Network::Socket::Type socket_type, uint32_t concurrency) { if (socket_type != Network::Socket::Type::Datagram) { diff --git a/source/server/listener_impl.h b/source/server/listener_impl.h index 2f0b860bd1332..bc77f42278ce4 100644 --- a/source/server/listener_impl.h +++ b/source/server/listener_impl.h @@ -57,7 +57,9 @@ class ListenSocketFactoryImpl : public Network::ListenSocketFactory, * @return the socket shared by worker threads; otherwise return null. */ Network::SocketOptRef sharedSocket() const override { - if (!reuse_port_) { + if (!reuse_port_ && + // internal address has no listener socket. + local_address_->envoyInternalAddress() == nullptr) { ASSERT(socket_ != nullptr); return *socket_; } @@ -305,6 +307,10 @@ class ListenerImpl final : public Network::ListenerConfig, return udp_listener_config_ != nullptr ? *udp_listener_config_ : Network::UdpListenerConfigOptRef(); } + Network::InternalListenerConfigOptRef internalListenerConfig() override { + return internal_listener_config_ != nullptr ? *internal_listener_config_ + : Network::InternalListenerConfigOptRef(); + } Network::ConnectionBalancer& connectionBalancer() override { return *connection_balancer_; } ResourceLimit& openConnections() override { return *open_connections_; } const std::vector& accessLogs() const override { @@ -351,6 +357,13 @@ class ListenerImpl final : public Network::ListenerConfig, Network::UdpListenerWorkerRouterPtr listener_worker_router_; }; + struct InternalListenerConfigImpl : public Network::InternalListenerConfig { + InternalListenerConfigImpl( + const envoy::config::listener::v3::Listener_InternalListenerConfig config) + : config_(config) {} + const envoy::config::listener::v3::Listener_InternalListenerConfig config_; + }; + /** * Create a new listener from an existing listener and the new config message if the in place * filter chain update is decided. Should be called only by newListenerWithFilterChain(). @@ -361,6 +374,7 @@ class ListenerImpl final : public Network::ListenerConfig, uint32_t concurrency); // Helpers for constructor. void buildAccessLog(); + void buildInternalListener(); void buildUdpListenerFactory(Network::Socket::Type socket_type, uint32_t concurrency); void buildListenSocketOptions(Network::Socket::Type socket_type); void createListenerFilterFactories(Network::Socket::Type socket_type); @@ -408,6 +422,7 @@ class ListenerImpl final : public Network::ListenerConfig, const std::chrono::milliseconds listener_filters_timeout_; const bool continue_on_listener_filters_timeout_; std::unique_ptr udp_listener_config_; + std::unique_ptr internal_listener_config_; Network::ConnectionBalancerSharedPtr connection_balancer_; std::shared_ptr listener_factory_context_; FilterChainManagerImpl filter_chain_manager_; diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index c9fa407fd7912..c398309e8965e 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -332,11 +332,13 @@ ListenerManagerStats ListenerManagerImpl::generateStats(Stats::Scope& scope) { bool ListenerManagerImpl::addOrUpdateListener(const envoy::config::listener::v3::Listener& config, const std::string& version_info, bool added_via_api) { - RELEASE_ASSERT( - !config.address().has_envoy_internal_address(), - fmt::format("listener {} has envoy internal address {}. Internal address cannot be used by " - "listener yet", - config.name(), config.address().envoy_internal_address().DebugString())); + if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.internal_address")) { + RELEASE_ASSERT( + !config.address().has_envoy_internal_address(), + fmt::format("listener {} has envoy internal address {}. Internal address cannot be used by " + "listener yet", + config.name(), config.address().envoy_internal_address().DebugString())); + } // TODO(junr03): currently only one ApiListener can be installed via bootstrap to avoid having to // build a collection of listeners, and to have to be able to warm and drain the listeners. In the // future allow multiple ApiListeners, and allow them to be created via LDS as well as bootstrap. diff --git a/source/server/worker_impl.cc b/source/server/worker_impl.cc index 68fd72defdb99..043a4e7a9cc1e 100644 --- a/source/server/worker_impl.cc +++ b/source/server/worker_impl.cc @@ -19,6 +19,7 @@ WorkerPtr ProdWorkerFactory::createWorker(uint32_t index, OverloadManager& overl Event::DispatcherPtr dispatcher( api_.allocateDispatcher(worker_name, overload_manager.scaledTimerFactory())); auto conn_handler = std::make_unique(*dispatcher, index); + dispatcher->registerInternalListenerManager(*conn_handler); return std::make_unique(tls_, hooks_, std::move(dispatcher), std::move(conn_handler), overload_manager, api_); } diff --git a/test/common/network/BUILD b/test/common/network/BUILD index e38f0aa74ff2e..87b676a17ecdf 100644 --- a/test/common/network/BUILD +++ b/test/common/network/BUILD @@ -91,6 +91,7 @@ envoy_cc_test( "//test/test_common:environment_lib", "//test/test_common:network_utility_lib", "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", "//test/test_common:test_time_lib", "//test/test_common:threadsafe_singleton_injector_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index b4109d17fb6fd..370f5c55fd61b 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -6,6 +6,7 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/event/scaled_range_timer_manager.h" #include "envoy/network/address.h" +#include "envoy/network/listener.h" #include "common/api/os_sys_calls_impl.h" #include "common/buffer/buffer_impl.h" @@ -29,6 +30,7 @@ #include "test/test_common/network_utility.h" #include "test/test_common/printers.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/threadsafe_singleton_injector.h" #include "test/test_common/utility.h" @@ -53,6 +55,12 @@ namespace Envoy { namespace Network { namespace { +class MockInternalListenerManager : public InternalListenerManager { +public: + MOCK_METHOD(InternalListenerCallbacksOptRef, findByAddress, + (const Address::InstanceConstSharedPtr&), ()); +}; + TEST(RawBufferSocket, TestBasics) { TransportSocketPtr raw_buffer_socket(Network::Test::createRawBufferSocket()); EXPECT_FALSE(raw_buffer_socket->ssl()); @@ -2950,6 +2958,41 @@ TEST_F(PipeClientConnectionImplTest, SkipSourceAddress) { connection->close(ConnectionCloseType::NoFlush); } +class InternalClientConnectionImplTest : public testing::Test { +protected: + InternalClientConnectionImplTest() + : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")) {} + + Api::ApiPtr api_; + Event::DispatcherPtr dispatcher_; + StrictMock client_callbacks_; +}; + +// Validate we skip setting source address. +TEST_F(InternalClientConnectionImplTest, SkipSourceAddress) { + auto scoped_runtime_guard = std::make_unique(); + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.internal_address", "true"}}); + + const Network::SocketInterface* sock_interface = Network::socketInterface( + "envoy.extensions.network.socket_interface.default_socket_interface"); + Network::Address::InstanceConstSharedPtr address = + std::make_shared("listener_0", sock_interface); + MockInternalListenerManager internal_listener_manager; + + dispatcher_->registerInternalListenerManager(internal_listener_manager); + EXPECT_CALL(internal_listener_manager, findByAddress(address)).WillOnce(Return(absl::nullopt)); + ClientConnectionPtr connection = + dispatcher_->createClientConnection(address, Network::Address::InstanceConstSharedPtr(), + Network::Test::createRawBufferSocket(), nullptr); + connection->addConnectionCallbacks(client_callbacks_); + connection->connect(); + EXPECT_CALL(client_callbacks_, onEvent(ConnectionEvent::RemoteClose)) + .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { dispatcher_->exit(); })); + dispatcher_->run(Event::Dispatcher::RunType::Block); + connection->close(ConnectionCloseType::NoFlush); +} + } // namespace } // namespace Network } // namespace Envoy diff --git a/test/config/utility.cc b/test/config/utility.cc index 0ce0a4313b5aa..6387c4822bec1 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -600,6 +600,18 @@ ConfigHelper::ConfigHelper(const Network::Address::IpVersion version, Api::Api& auto* static_resources = bootstrap_.mutable_static_resources(); for (int i = 0; i < static_resources->listeners_size(); ++i) { auto* listener = static_resources->mutable_listeners(i); + if (listener->mutable_address()->has_envoy_internal_address()) { + ENVOY_LOG_MISC( + debug, "Listener {} has internal address {}. Will not reset to loop back socket address.", + i, listener->mutable_address()->envoy_internal_address().server_listener_name()); + continue; + } + if (listener->mutable_address()->has_pipe()) { + ENVOY_LOG_MISC(debug, + "Listener {} has pipe address {}. Will not reset to loop back socket address.", + i, listener->mutable_address()->pipe().path()); + continue; + } auto* listener_socket_addr = listener->mutable_address()->mutable_socket_address(); if (listener_socket_addr->address() == "0.0.0.0" || listener_socket_addr->address() == "::") { listener_socket_addr->set_address(Network::Test::getAnyAddressString(version)); @@ -941,7 +953,6 @@ void ConfigHelper::setDefaultHostAndRoute(const std::string& domains, const std: void ConfigHelper::setBufferLimits(uint32_t upstream_buffer_limit, uint32_t downstream_buffer_limit) { RELEASE_ASSERT(!finalized_, ""); - RELEASE_ASSERT(bootstrap_.mutable_static_resources()->listeners_size() == 1, ""); auto* listener = bootstrap_.mutable_static_resources()->mutable_listeners(0); listener->mutable_per_connection_buffer_limit_bytes()->set_value(downstream_buffer_limit); const uint32_t stream_buffer_size = std::max( diff --git a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc index 537665926a91c..c6bed1213f167 100644 --- a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc +++ b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc @@ -74,6 +74,9 @@ class ProxyProtocolRegressionTest : public testing::TestWithParam { upstream_config_.http2_options_.MergeFrom(options); } + Event::Dispatcher* getWorkerDispatcher(uint32_t index) { + if (test_server_ == nullptr) { + return nullptr; + } + return &test_server_->getWorkerDispatcher(index).value().get(); + } + std::unique_ptr upstream_stats_store_; // Make sure the test server will be torn down after any fake client. diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index eb3e9ac757095..629037857f604 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -13,6 +13,7 @@ #include "envoy/network/connection.h" #include "envoy/network/connection_handler.h" #include "envoy/network/filter.h" +#include "envoy/network/listener.h" #include "envoy/stats/scope.h" #include "common/buffer/buffer_impl.h" @@ -769,6 +770,9 @@ class FakeUpstream : Logger::Loggable, uint64_t listenerTag() const override { return 0; } const std::string& name() const override { return name_; } Network::UdpListenerConfigOptRef udpListenerConfig() override { return udp_listener_config_; } + Network::InternalListenerConfigOptRef internalListenerConfig() override { + return Network::InternalListenerConfigOptRef(); + } Network::ConnectionBalancer& connectionBalancer() override { return connection_balancer_; } envoy::config::core::v3::TrafficDirection direction() const override { return envoy::config::core::v3::UNSPECIFIED; diff --git a/test/integration/integration_tcp_client.cc b/test/integration/integration_tcp_client.cc index aabf2a957d956..f6356723eebd6 100644 --- a/test/integration/integration_tcp_client.cc +++ b/test/integration/integration_tcp_client.cc @@ -16,6 +16,13 @@ #include "common/common/fmt.h" #include "common/network/utility.h" +/* begin internal connection */ +#include "common/stream_info/stream_info_impl.h" +#include "common/network/listen_socket_impl.h" +#include "common/network/connection_impl.h" +#include "extensions/io_socket/user_space/io_handle_impl.h" +/* end internal connection */ + #include "test/integration/utility.h" #include "test/mocks/buffer/mocks.h" #include "test/test_common/network_utility.h" @@ -53,7 +60,6 @@ IntegrationTcpClient::IntegrationTcpClient( std::function above_overflow) -> Buffer::Instance* { return new Buffer::WatermarkBuffer(below_low, above_high, above_overflow); })); - ; connection_ = dispatcher.createClientConnection( Network::Utility::resolveUrl( @@ -70,6 +76,44 @@ IntegrationTcpClient::IntegrationTcpClient( connection_->connect(); } +IntegrationTcpClient::IntegrationTcpClient(Event::Dispatcher& dispatcher, + MockBufferFactory& factory, + Network::Address::InstanceConstSharedPtr, + bool enable_half_close, + const Network::ConnectionSocket::OptionsSharedPtr&, + Network::Address::InstanceConstSharedPtr) + : payload_reader_(new WaitForPayloadReader(dispatcher)), + callbacks_(new ConnectionCallbacks(*this)) { + EXPECT_CALL(factory, create_(_, _, _)) + .Times(AtLeast(1)) + .WillOnce(Invoke([&](std::function below_low, std::function above_high, + std::function above_overflow) -> Buffer::Instance* { + client_write_buffer_ = + new NiceMock(below_low, above_high, above_overflow); + return client_write_buffer_; + })) + .WillRepeatedly(Invoke([](std::function below_low, std::function above_high, + std::function above_overflow) -> Buffer::Instance* { + return new Buffer::WatermarkBuffer(below_low, above_high, above_overflow); + })); + auto [io_handle_client, io_handle_server] = + Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); + StreamInfo::StreamInfoImpl stream_info(dispatcher.timeSource(), nullptr); + Network::ConnectionImpl client_connection(dispatcher, + std::make_unique( + std::move(io_handle_client), nullptr, nullptr), + Network::Test::createRawBufferSocket(), stream_info, + false); + + ON_CALL(*client_write_buffer_, drain(_)) + .WillByDefault(Invoke(client_write_buffer_, &MockWatermarkBuffer::trackDrains)); + EXPECT_CALL(*client_write_buffer_, drain(_)).Times(AnyNumber()); + + connection_->enableHalfClose(enable_half_close); + connection_->addConnectionCallbacks(*callbacks_); + connection_->addReadFilter(payload_reader_); +} + void IntegrationTcpClient::close() { connection_->close(Network::ConnectionCloseType::NoFlush); } void IntegrationTcpClient::waitForData(const std::string& data, bool exact_match) { diff --git a/test/integration/integration_tcp_client.h b/test/integration/integration_tcp_client.h index af75b84de08c1..a14c5cd9b4ceb 100644 --- a/test/integration/integration_tcp_client.h +++ b/test/integration/integration_tcp_client.h @@ -30,6 +30,11 @@ class IntegrationTcpClient { Network::Address::IpVersion version, bool enable_half_close, const Network::ConnectionSocket::OptionsSharedPtr& options, Network::Address::InstanceConstSharedPtr source_address = nullptr); + IntegrationTcpClient(Event::Dispatcher& dispatcher, MockBufferFactory& factory, + Network::Address::InstanceConstSharedPtr dest_address, + bool enable_half_close, + const Network::ConnectionSocket::OptionsSharedPtr& options, + Network::Address::InstanceConstSharedPtr source_address = nullptr); void close(); void waitForData(const std::string& data, bool exact_match = true); diff --git a/test/integration/internal_tcp_proxy_integration_test.cc b/test/integration/internal_tcp_proxy_integration_test.cc new file mode 100644 index 0000000000000..fad3a753d9864 --- /dev/null +++ b/test/integration/internal_tcp_proxy_integration_test.cc @@ -0,0 +1,903 @@ +#include + +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.pb.h" +#include "envoy/extensions/access_loggers/file/v3/file.pb.h" +#include "envoy/extensions/filters/network/tcp_proxy/v3/tcp_proxy.pb.h" +#include "envoy/network/address.h" + +#include "common/config/api_version.h" +#include "common/network/address_impl.h" +#include "common/network/utility.h" + +#include "extensions/filters/network/common/factory_base.h" +#include "extensions/io_socket/user_space/io_handle_impl.h" +#include "extensions/transport_sockets/tls/context_manager_impl.h" + +#include "test/integration/ssl_utility.h" +#include "test/integration/tcp_proxy_integration_test.h" +#include "test/integration/tcp_proxy_integration_test.pb.h" +#include "test/integration/tcp_proxy_integration_test.pb.validate.h" +#include "test/integration/utility.h" +#include "test/test_common/registry.h" + +#include "gtest/gtest.h" + +using testing::HasSubstr; + +namespace Envoy { + +class InternalTcpProxyIntegrationTest + : public testing::TestWithParam, + public BaseIntegrationTest { +public: + InternalTcpProxyIntegrationTest() + : BaseIntegrationTest(GetParam().version, internalTcpProxyConfig()) { + enableHalfClose(true); + } + static std::string internalTcpProxyConfig() { + return absl::StrCat(fmt::format(R"EOF( +admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: "{}" + address: + socket_address: + address: 127.0.0.1 + port_value: 0 +dynamic_resources: + lds_config: + resource_api_version: V3 + path: {} +static_resources: + secrets: + - name: "secret_static_0" + tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES" + private_key: + inline_string: "DUMMY_INLINE_BYTES" + password: + inline_string: "DUMMY_INLINE_BYTES" + clusters: + - name: cluster_0 + load_assignment: + cluster_name: cluster_0 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + - name: cluster_internal + load_assignment: + cluster_name: cluster_internal + endpoints: + - lb_endpoints: + - endpoint: + address: +# pipe: +# path: "@/tmp/pipetest" + envoy_internal_address: + server_listener_name: test_internal_listener_foo + + listeners: + - name: listener_0 + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + filter_chains: + filters: + name: tcp + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp_stats + cluster: cluster_internal + - name: listener_internal + address: +# pipe: +# path: "@/tmp/pipetest" + envoy_internal_address: + server_listener_name: test_internal_listener_foo + internal_listener: {{ }} # escape the bracket +)EOF", + Platform::null_device_path, Platform::null_device_path), + R"EOF( + filter_chains: + filters: + name: tcp + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp_stats + cluster: cluster_0 +)EOF"); + } + + void initialize() override; + + Event::Dispatcher* worker_dispatcher_{}; + int chained_number_{2}; +}; + +std::vector newPoolTestParams() { + std::vector ret; + + for (auto ip_version : TestEnvironment::getIpVersionsForTest()) { + ret.push_back(TcpProxyIntegrationTestParams{ip_version, false}); + } + return ret; +} + +std::vector getProtocolTestParams() { + std::vector ret; + + for (auto ip_version : TestEnvironment::getIpVersionsForTest()) { + ret.push_back(TcpProxyIntegrationTestParams{ip_version, true}); + ret.push_back(TcpProxyIntegrationTestParams{ip_version, false}); + } + return ret; +} + +std::string +protocolTestParamsToString(const ::testing::TestParamInfo& params) { + return absl::StrCat( + (params.param.version == Network::Address::IpVersion::v4 ? "IPv4_" : "IPv6_"), + (params.param.test_original_version == true ? "OriginalConnPool" : "NewConnPool")); +} + +void InternalTcpProxyIntegrationTest::initialize() { + if (GetParam().test_original_version) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.new_tcp_connection_pool", "false"); + } else { + config_helper_.addRuntimeOverride("envoy.reloadable_features.new_tcp_connection_pool", "true"); + } + + config_helper_.renameListener("tcp_proxy"); + BaseIntegrationTest::initialize(); + + // worker_dispatcher_ can only be obtained after initialization. + worker_dispatcher_ = getWorkerDispatcher(1); + ASSERT(worker_dispatcher_ != nullptr); +} + +// The internal connection clients are driven by the dispatcher. +INSTANTIATE_TEST_SUITE_P(TcpProxyIntegrationTestParams, InternalTcpProxyIntegrationTest, + testing::ValuesIn(getProtocolTestParams()), protocolTestParamsToString); + +// There are two tcp proxies in this test suite. +// +// The first tcp proxy is owned by the tcp listener. This tcp proxy handles the data from test main +// thread and create upstream internal connection to the internal listener. +// +// The second tcp proxy is owned by the internal listener. This tcp proxy relay the data to the fake +// upstream. +using ChainedProxyInternalTcpProxyIntegrationTest = InternalTcpProxyIntegrationTest; +INSTANTIATE_TEST_SUITE_P(TcpProxyIntegrationTestParams, ChainedProxyInternalTcpProxyIntegrationTest, + testing::ValuesIn(getProtocolTestParams()), protocolTestParamsToString); + +// TODO(lambdai): enable it when the io_handle_client can be gracefully destroyed in the worker +// thread. +TEST_P(InternalTcpProxyIntegrationTest, DISABLED_TcpProxyUpstreamWritesFirst) { + initialize(); + + // lambda-expression cannot captures a structured binding so we must use tie() here. + Network::IoHandlePtr io_handle_client; + Network::IoHandlePtr io_handle_server; + + std::tie(io_handle_client, io_handle_server) = + Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); + + auto server_address = + std::make_shared("test_internal_listener_foo"); + auto client_address = std::make_shared("client_bar"); + + worker_dispatcher_->post([&]() mutable { + auto internal_listener = + worker_dispatcher_->getInternalListenerManagerForTest().value().get().findByAddress( + server_address); + std::unique_ptr io_handle = std::move(io_handle_server); + auto accepted_socket = std::make_unique( + std::move(io_handle), server_address, client_address); + internal_listener.value().get().onAccept(std::move(accepted_socket)); + }); + + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + + ASSERT_TRUE(fake_upstream_connection->write("hello")); + + Buffer::OwnedImpl buffer; + Thread::MutexBasicLockable dispatcher_mutex; + + while (buffer.length() < 5) { + Thread::CondVar post_complete; + worker_dispatcher_->post([&]() { + Thread::LockGuard guard(dispatcher_mutex); + io_handle_client->read(buffer, absl::nullopt); + post_complete.notifyOne(); + }); + + Thread::LockGuard guard(dispatcher_mutex); + post_complete.wait(dispatcher_mutex); + ENVOY_LOG_MISC(debug, "current buffer: {}", buffer.toString()); + } + ASSERT_TRUE(fake_upstream_connection->write("", true)); +} + +// Test upstream writing before downstream downstream does. +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyUpstreamWritesFirst) { + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + + ASSERT_TRUE(fake_upstream_connection->write("hello")); + tcp_client->waitForData("hello"); + // Make sure inexact matches work also on data already received. + tcp_client->waitForData("ello", false); + + // Make sure length based wait works for the data already received + ASSERT_TRUE(tcp_client->waitForData(5)); + ASSERT_TRUE(tcp_client->waitForData(4)); + + // Drain part of the received message + tcp_client->clearData(2); + tcp_client->waitForData("llo"); + ASSERT_TRUE(tcp_client->waitForData(3)); + + ASSERT_TRUE(tcp_client->write("hello")); + ASSERT_TRUE(fake_upstream_connection->waitForData(5)); + + ASSERT_TRUE(fake_upstream_connection->write("", true)); + tcp_client->waitForHalfClose(); + ASSERT_TRUE(tcp_client->write("", true)); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + // Any time an associated connection is destroyed, it increments both counters. + test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_destroy", 1); + test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_destroy_with_active_rq", 1); + + IntegrationTcpClientPtr tcp_client2 = makeTcpConnection(lookupPort("tcp_proxy")); + FakeRawConnectionPtr fake_upstream_connection2; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection2)); + tcp_client2->close(); +} + +// Test TLS upstream. +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyUpstreamTls) { + upstream_tls_ = true; + setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); + config_helper_.configureUpstreamTls(); + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + ASSERT_TRUE(tcp_client->write("hello")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT_TRUE(fake_upstream_connection->waitForData(5)); + ASSERT_TRUE(fake_upstream_connection->write("world")); + ASSERT_TRUE(fake_upstream_connection->close()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + tcp_client->waitForHalfClose(); + tcp_client->close(); + + EXPECT_EQ("world", tcp_client->data()); + // Any time an associated connection is destroyed, it increments both counters. + test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_destroy", 1); + test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_destroy_with_active_rq", 1); +} + +// Test proxying data in both directions, and that all data is flushed properly +// when there is an upstream disconnect. +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyUpstreamDisconnect) { + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + ASSERT_TRUE(tcp_client->write("hello")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT_TRUE(fake_upstream_connection->waitForData(5)); + ASSERT_TRUE(fake_upstream_connection->write("world")); + ASSERT_TRUE(fake_upstream_connection->close()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + tcp_client->waitForHalfClose(); + tcp_client->close(); + + EXPECT_EQ("world", tcp_client->data()); +} + +// Test proxying data in both directions, and that all data is flushed properly +// when the client disconnects. +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyDownstreamDisconnect) { + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + ASSERT_TRUE(tcp_client->write("hello")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT_TRUE(fake_upstream_connection->waitForData(5)); + ASSERT_TRUE(fake_upstream_connection->write("world")); + tcp_client->waitForData("world"); + ASSERT_TRUE(tcp_client->write("hello", true)); + ASSERT_TRUE(fake_upstream_connection->waitForData(10)); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->write("", true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + tcp_client->waitForDisconnect(); +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyManyConnections) { + autonomous_upstream_ = true; + initialize(); + const int num_connections = 50; + std::vector clients(num_connections); + + for (int i = 0; i < num_connections; ++i) { + clients[i] = makeTcpConnection(lookupPort("tcp_proxy")); + } + test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_total", num_connections); + for (int i = 0; i < num_connections; ++i) { + IntegrationTcpClientPtr& tcp_client = clients[i]; + // The autonomous upstream is an HTTP upstream, so send raw HTTP. + // This particular request will result in the upstream sending a response, + // and flush-closing due to the 'close_after_response' header. + ASSERT_TRUE(tcp_client->write( + "GET / HTTP/1.1\r\nHost: foo\r\nclose_after_response: yes\r\ncontent-length: 0\r\n\r\n", + false)); + tcp_client->waitForHalfClose(); + tcp_client->close(); + EXPECT_THAT(tcp_client->data(), HasSubstr("aaaaaaaaaa")); + } +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyRandomBehavior) { + autonomous_upstream_ = true; + initialize(); + std::list clients; + + // The autonomous upstream parses HTTP, and HTTP headers and sends responses + // when full requests are received. basic_request will result in + // bidirectional data. request_with_close will result in bidirectional data, + // but also the upstream closing the connection. + const char* basic_request = "GET / HTTP/1.1\r\nHost: foo\r\ncontent-length: 0\r\n\r\n"; + const char* request_with_close = + "GET / HTTP/1.1\r\nHost: foo\r\nclose_after_response: yes\r\ncontent-length: 0\r\n\r\n"; + TestRandomGenerator rand; + + // Seed some initial clients + for (int i = 0; i < 5; ++i) { + clients.push_back(makeTcpConnection(lookupPort("tcp_proxy"))); + } + + // Now randomly write / add more connections / close. + for (int i = 0; i < 50; ++i) { + int action = rand.random() % 3; + + if (action == 0) { + // Add a new connection. + clients.push_back(makeTcpConnection(lookupPort("tcp_proxy"))); + } + if (clients.empty()) { + break; + } + IntegrationTcpClientPtr& tcp_client = clients.front(); + if (action == 1) { + // Write to the first connection. + ASSERT_TRUE(tcp_client->write(basic_request, false)); + tcp_client->waitForData("\r\n\r\n", false); + tcp_client->clearData(tcp_client->data().size()); + } else if (action == 2) { + // Close the first connection. + ASSERT_TRUE(tcp_client->write(request_with_close, false)); + tcp_client->waitForData("\r\n\r\n", false); + tcp_client->waitForHalfClose(); + tcp_client->close(); + clients.pop_front(); + } + } + + while (!clients.empty()) { + IntegrationTcpClientPtr& tcp_client = clients.front(); + ASSERT_TRUE(tcp_client->write(request_with_close, false)); + tcp_client->waitForData("\r\n\r\n", false); + tcp_client->waitForHalfClose(); + tcp_client->close(); + clients.pop_front(); + } +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, NoUpstream) { + // Set the first upstream to have an invalid port, so connection will fail, + // but it won't fail synchronously (as it would if there were simply no + // upstreams) + fake_upstreams_count_ = 0; + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* cluster = bootstrap.mutable_static_resources()->mutable_clusters(0); + auto* lb_endpoint = + cluster->mutable_load_assignment()->mutable_endpoints(0)->mutable_lb_endpoints(0); + lb_endpoint->mutable_endpoint()->mutable_address()->mutable_socket_address()->set_port_value(1); + }); + config_helper_.skipPortUsageValidation(); + enableHalfClose(false); + initialize(); + + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + tcp_client->waitForDisconnect(); +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyLargeWrite) { + config_helper_.setBufferLimits(1024, 1024); + initialize(); + + std::string data(1024 * 16, 'a'); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + ASSERT_TRUE(tcp_client->write(data)); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT_TRUE(fake_upstream_connection->waitForData(data.size())); + ASSERT_TRUE(fake_upstream_connection->write(data)); + tcp_client->waitForData(data); + tcp_client->close(); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->close()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + + uint32_t upstream_pauses = + test_server_->counter("cluster.cluster_0.upstream_flow_control_paused_reading_total") + ->value(); + uint32_t upstream_resumes = + test_server_->counter("cluster.cluster_0.upstream_flow_control_resumed_reading_total") + ->value(); + EXPECT_EQ(upstream_pauses, upstream_resumes); + + uint32_t downstream_pauses = + test_server_->counter("tcp.tcp_stats.downstream_flow_control_paused_reading_total")->value(); + uint32_t downstream_resumes = + test_server_->counter("tcp.tcp_stats.downstream_flow_control_resumed_reading_total")->value(); + EXPECT_EQ(downstream_pauses, downstream_resumes); +} + +// Test that a downstream flush works correctly (all data is flushed) +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyDownstreamFlush) { + // Use a very large size to make sure it is larger than the kernel socket read buffer. + const uint32_t size = 50 * 1024 * 1024; + config_helper_.setBufferLimits(size / 8, size / 8); + initialize(); + + std::string data(size, 'a'); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + tcp_client->readDisable(true); + ASSERT_TRUE(tcp_client->write("", true)); + + // This ensures that readDisable(true) has been run on it's thread + // before tcp_client starts writing. + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + + ASSERT_TRUE(fake_upstream_connection->write(data, true)); + + // test_server_->waitForCounterGe("cluster.cluster_0.upstream_flow_control_paused_reading_total", + // 1); + // EXPECT_EQ(test_server_->counter("cluster.cluster_0.upstream_flow_control_resumed_reading_total") + // ->value(), + // 0); + tcp_client->readDisable(false); + tcp_client->waitForData(data); + tcp_client->waitForHalfClose(); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + + uint32_t upstream_pauses = + test_server_->counter("cluster.cluster_0.upstream_flow_control_paused_reading_total") + ->value(); + uint32_t upstream_resumes = + test_server_->counter("cluster.cluster_0.upstream_flow_control_resumed_reading_total") + ->value(); + EXPECT_GE(upstream_pauses, upstream_resumes); + EXPECT_GT(upstream_resumes, 0); +} + +// Test that an upstream flush works correctly (all data is flushed) +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyUpstreamFlush) { + // Use a very large size to make sure it is larger than the kernel socket read buffer. + const uint32_t size = 1 * 1024 * 1024; + // config_helper_.setBufferLimits(size / 2, size / 2); + initialize(); + + std::string data(size, 'a'); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT_TRUE(fake_upstream_connection->readDisable(true)); + ASSERT_TRUE(fake_upstream_connection->write("", true)); + + // This ensures that fake_upstream_connection->readDisable has been run on it's thread + // before tcp_client starts writing. + tcp_client->waitForHalfClose(); + + ASSERT_TRUE(tcp_client->write(data, true, true, std::chrono::milliseconds(30000))); + FANCY_LOG(debug, "lambdai: waiting for upstream_flush_active"); + // test_server_->waitForGaugeGe("tcp.tcp_stats.upstream_flush_active", 1); + ASSERT_TRUE(fake_upstream_connection->readDisable(false)); + ASSERT_TRUE(fake_upstream_connection->waitForData(data.size())); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + // tcp_client->waitForDisconnect(true); + FANCY_LOG(debug, "client disconnected"); + // Double because two tcp proxy are chained. + EXPECT_EQ(test_server_->counter("tcp.tcp_stats.upstream_flush_total")->value(), chained_number_); + FANCY_LOG(debug, "flush total == {} ", + test_server_->counter("tcp.tcp_stats.upstream_flush_total")->value()); + test_server_->waitForGaugeEq("tcp.tcp_stats.upstream_flush_active", 0); +} + +// Test that Envoy doesn't crash or assert when shutting down with an upstream flush active +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyUpstreamFlushEnvoyExit) { + // Use a very large size to make sure it is larger than the kernel socket read buffer. + const uint32_t size = 50 * 1024 * 1024; + config_helper_.setBufferLimits(size, size); + initialize(); + + std::string data(size, 'a'); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT_TRUE(fake_upstream_connection->readDisable(true)); + ASSERT_TRUE(fake_upstream_connection->write("", true)); + + // This ensures that fake_upstream_connection->readDisable has been run on it's thread + // before tcp_client starts writing. + tcp_client->waitForHalfClose(); + + ASSERT_TRUE(tcp_client->write(data, true)); + + test_server_->waitForGaugeEq("tcp.tcp_stats.upstream_flush_active", 1); + test_server_.reset(); + ASSERT_TRUE(fake_upstream_connection->close()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + + // Success criteria is that no ASSERTs fire and there are no leaks. +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, AccessLog) { + std::string access_log_path = TestEnvironment::temporaryPath( + fmt::format("access_log{}.txt", version_ == Network::Address::IpVersion::v4 ? "v4" : "v6")); + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* filter_chain = listener->mutable_filter_chains(0); + auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); + + ASSERT_TRUE(config_blob->Is()); + auto tcp_proxy_config = + MessageUtil::anyConvert( + *config_blob); + + auto* access_log = tcp_proxy_config.add_access_log(); + access_log->set_name("accesslog"); + envoy::extensions::access_loggers::file::v3::FileAccessLog access_log_config; + access_log_config.set_path(access_log_path); + access_log_config.mutable_log_format()->mutable_text_format_source()->set_inline_string( + "upstreamlocal=%UPSTREAM_LOCAL_ADDRESS% " + "upstreamhost=%UPSTREAM_HOST% downstream=%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT% " + "sent=%BYTES_SENT% received=%BYTES_RECEIVED%\n"); + access_log->mutable_typed_config()->PackFrom(access_log_config); + auto* runtime_filter = access_log->mutable_filter()->mutable_runtime_filter(); + runtime_filter->set_runtime_key("unused-key"); + auto* percent_sampled = runtime_filter->mutable_percent_sampled(); + percent_sampled->set_numerator(100); + percent_sampled->set_denominator(envoy::type::v3::FractionalPercent::DenominatorType:: + FractionalPercent_DenominatorType_HUNDRED); + config_blob->PackFrom(tcp_proxy_config); + }); + initialize(); + + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + + ASSERT_TRUE(fake_upstream_connection->write("hello")); + tcp_client->waitForData("hello"); + + ASSERT_TRUE(fake_upstream_connection->write("", true)); + tcp_client->waitForHalfClose(); + ASSERT_TRUE(tcp_client->write("", true)); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + + std::string log_result; + // Access logs only get flushed to disk periodically, so poll until the log is non-empty + do { + log_result = api_->fileSystem().fileReadToEnd(access_log_path); + } while (log_result.empty()); + + // Regex matching localhost:port +#ifndef GTEST_USES_SIMPLE_RE + const std::string ip_port_regex = (version_ == Network::Address::IpVersion::v4) + ? R"EOF(127\.0\.0\.1:[0-9]+)EOF" + : R"EOF(\[::1\]:[0-9]+)EOF"; +#else + const std::string ip_port_regex = (version_ == Network::Address::IpVersion::v4) + ? R"EOF(127\.0\.0\.1:\d+)EOF" + : R"EOF(\[::1\]:\d+)EOF"; +#endif + + const std::string ip_regex = + (version_ == Network::Address::IpVersion::v4) ? R"EOF(127.0.0.1)EOF" : R"EOF(::1)EOF"; + + // Test that all three addresses were populated correctly. Only check the first line of + // log output for simplicity. + EXPECT_THAT(log_result, HasSubstr(fmt::format( + "upstreamlocal={0} upstreamhost={2} downstream={1} sent=5 received=0", + // Upstream local is not defined for internal connection. + "-", ip_regex, + // Upstream remote address is formatted as internal listener name. + "envoy://test_internal_listener_foo"))); +} + +// Test that the server shuts down without crashing when connections are open. +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, ShutdownWithOpenConnections) { + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* static_resources = bootstrap.mutable_static_resources(); + for (int i = 0; i < static_resources->clusters_size(); ++i) { + auto* cluster = static_resources->mutable_clusters(i); + cluster->set_close_connections_on_host_health_failure(true); + } + }); + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + ASSERT_TRUE(tcp_client->write("hello")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT_TRUE(fake_upstream_connection->waitForData(5)); + ASSERT_TRUE(fake_upstream_connection->write("world")); + tcp_client->waitForData("world"); + ASSERT_TRUE(tcp_client->write("hello", false)); + ASSERT_TRUE(fake_upstream_connection->waitForData(10)); + test_server_.reset(); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->close()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + tcp_client->waitForHalfClose(); + tcp_client->close(); + + // Success criteria is that no ASSERTs fire and there are no leaks. +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TestIdletimeoutWithNoData) { + autonomous_upstream_ = true; + + enableHalfClose(false); + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* filter_chain = listener->mutable_filter_chains(0); + auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); + + ASSERT_TRUE(config_blob->Is()); + auto tcp_proxy_config = + MessageUtil::anyConvert( + *config_blob); + tcp_proxy_config.mutable_idle_timeout()->set_nanos( + std::chrono::duration_cast(std::chrono::milliseconds(100)) + .count()); + config_blob->PackFrom(tcp_proxy_config); + }); + + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + tcp_client->waitForDisconnect(); +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TestIdletimeoutWithLargeOutstandingData) { + config_helper_.setBufferLimits(1024, 1024); + enableHalfClose(false); + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* filter_chain = listener->mutable_filter_chains(0); + auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); + + ASSERT_TRUE(config_blob->Is()); + auto tcp_proxy_config = + MessageUtil::anyConvert( + *config_blob); + tcp_proxy_config.mutable_idle_timeout()->set_nanos( + std::chrono::duration_cast(std::chrono::milliseconds(500)) + .count()); + config_blob->PackFrom(tcp_proxy_config); + }); + + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + + std::string data(1024 * 16, 'a'); + ASSERT_TRUE(tcp_client->write(data)); + ASSERT_TRUE(fake_upstream_connection->write(data)); + + tcp_client->waitForDisconnect(); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TestMaxDownstreamConnectionDurationWithNoData) { + autonomous_upstream_ = true; + + enableHalfClose(false); + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* filter_chain = listener->mutable_filter_chains(0); + auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); + + ASSERT_TRUE(config_blob->Is()); + auto tcp_proxy_config = + MessageUtil::anyConvert( + *config_blob); + tcp_proxy_config.mutable_max_downstream_connection_duration()->set_nanos( + std::chrono::duration_cast(std::chrono::milliseconds(100)) + .count()); + config_blob->PackFrom(tcp_proxy_config); + }); + + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + tcp_client->waitForDisconnect(); +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, + TestMaxDownstreamConnectionDurationWithLargeOutstandingData) { + config_helper_.setBufferLimits(1024, 1024); + enableHalfClose(false); + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* filter_chain = listener->mutable_filter_chains(0); + auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); + + ASSERT_TRUE(config_blob->Is()); + auto tcp_proxy_config = + MessageUtil::anyConvert( + *config_blob); + tcp_proxy_config.mutable_max_downstream_connection_duration()->set_nanos( + std::chrono::duration_cast(std::chrono::milliseconds(500)) + .count()); + config_blob->PackFrom(tcp_proxy_config); + }); + + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + + std::string data(1024 * 16, 'a'); + ASSERT_TRUE(tcp_client->write(data)); + ASSERT_TRUE(fake_upstream_connection->write(data)); + + tcp_client->waitForDisconnect(); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, DISABLED_TestNoCloseOnHealthFailure) { + concurrency_ = 2; + + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* static_resources = bootstrap.mutable_static_resources(); + for (int i = 0; i < static_resources->clusters_size(); ++i) { + auto* cluster = static_resources->mutable_clusters(i); + cluster->set_close_connections_on_host_health_failure(false); + cluster->mutable_common_lb_config()->mutable_healthy_panic_threshold()->set_value(0); + cluster->add_health_checks()->mutable_timeout()->set_seconds(20); + cluster->mutable_health_checks(0)->mutable_reuse_connection()->set_value(true); + cluster->mutable_health_checks(0)->mutable_interval()->set_seconds(1); + cluster->mutable_health_checks(0)->mutable_no_traffic_interval()->set_seconds(1); + cluster->mutable_health_checks(0)->mutable_unhealthy_threshold()->set_value(1); + cluster->mutable_health_checks(0)->mutable_healthy_threshold()->set_value(1); + cluster->mutable_health_checks(0)->mutable_tcp_health_check(); + cluster->mutable_health_checks(0)->mutable_tcp_health_check()->mutable_send()->set_text( + "50696E67"); + cluster->mutable_health_checks(0)->mutable_tcp_health_check()->add_receive()->set_text( + "506F6E67"); + } + }); + + FakeRawConnectionPtr fake_upstream_health_connection; + on_server_init_function_ = [&](void) -> void { + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_health_connection)); + ASSERT_TRUE(fake_upstream_health_connection->waitForData( + FakeRawConnection::waitForInexactMatch("Ping"))); + ASSERT_TRUE(fake_upstream_health_connection->write("Pong")); + }; + + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + ASSERT_TRUE(tcp_client->write("hello")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT_TRUE(fake_upstream_connection->waitForData(5)); + ASSERT_TRUE(fake_upstream_connection->write("world")); + tcp_client->waitForData("world"); + ASSERT_TRUE(tcp_client->write("hello")); + ASSERT_TRUE(fake_upstream_connection->waitForData(10)); + + ASSERT_TRUE(fake_upstream_health_connection->waitForData(8)); + ASSERT_TRUE(fake_upstream_health_connection->close()); + ASSERT_TRUE(fake_upstream_health_connection->waitForDisconnect()); + + // By waiting we know the previous health check attempt completed (with a failure since we closed + // the connection on it) + FakeRawConnectionPtr fake_upstream_health_connection_reconnect; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_health_connection_reconnect)); + ASSERT_TRUE(fake_upstream_health_connection_reconnect->waitForData( + FakeRawConnection::waitForInexactMatch("Ping"))); + + ASSERT_TRUE(tcp_client->write("still")); + ASSERT_TRUE(fake_upstream_connection->waitForData(15)); + ASSERT_TRUE(fake_upstream_connection->write("here")); + tcp_client->waitForData("here", false); + + test_server_.reset(); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->close()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + ASSERT_TRUE(fake_upstream_health_connection_reconnect->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_health_connection_reconnect->close()); + ASSERT_TRUE(fake_upstream_health_connection_reconnect->waitForDisconnect()); + tcp_client->waitForHalfClose(); + tcp_client->close(); +} + +TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, DISABLED_TestCloseOnHealthFailure) { + concurrency_ = 2; + + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* static_resources = bootstrap.mutable_static_resources(); + for (int i = 0; i < static_resources->clusters_size(); ++i) { + auto* cluster = static_resources->mutable_clusters(i); + cluster->set_close_connections_on_host_health_failure(true); + cluster->mutable_common_lb_config()->mutable_healthy_panic_threshold()->set_value(0); + cluster->add_health_checks()->mutable_timeout()->set_seconds(20); + cluster->mutable_health_checks(0)->mutable_reuse_connection()->set_value(true); + cluster->mutable_health_checks(0)->mutable_interval()->set_seconds(1); + cluster->mutable_health_checks(0)->mutable_no_traffic_interval()->set_seconds(1); + cluster->mutable_health_checks(0)->mutable_unhealthy_threshold()->set_value(1); + cluster->mutable_health_checks(0)->mutable_healthy_threshold()->set_value(1); + cluster->mutable_health_checks(0)->mutable_tcp_health_check(); + cluster->mutable_health_checks(0)->mutable_tcp_health_check()->mutable_send()->set_text( + "50696E67"); + ; + cluster->mutable_health_checks(0)->mutable_tcp_health_check()->add_receive()->set_text( + "506F6E67"); + } + }); + + FakeRawConnectionPtr fake_upstream_health_connection; + on_server_init_function_ = [&](void) -> void { + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_health_connection)); + ASSERT_TRUE(fake_upstream_health_connection->waitForData(4)); + ASSERT_TRUE(fake_upstream_health_connection->write("Pong")); + }; + + initialize(); + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + ASSERT_TRUE(tcp_client->write("hello")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + ASSERT_TRUE(fake_upstream_connection->waitForData(5)); + ASSERT_TRUE(fake_upstream_connection->write("world")); + tcp_client->waitForData("world"); + ASSERT_TRUE(tcp_client->write("hello")); + ASSERT_TRUE(fake_upstream_connection->waitForData(10)); + + ASSERT_TRUE(fake_upstream_health_connection->waitForData(8)); + ASSERT_TRUE(fake_upstream_health_connection->close()); + ASSERT_TRUE(fake_upstream_health_connection->waitForDisconnect()); + + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + tcp_client->waitForHalfClose(); + + ASSERT_TRUE(fake_upstream_connection->close()); + tcp_client->close(); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); +} + +} // namespace Envoy diff --git a/test/integration/server.cc b/test/integration/server.cc index 95c3ad4452635..b071e9713d436 100644 --- a/test/integration/server.cc +++ b/test/integration/server.cc @@ -242,6 +242,16 @@ void IntegrationTestServerImpl::createAndRunEnvoyServer( admin_address_ = server.admin().socket().addressProvider().localAddress(); server_ = &server; stat_store_ = &stat_store; + // Use the tls slots to save dispatchers. The slot must be destroyed before server shutdown. + ThreadLocal::SlotPtr dispatcher_tls = tls.allocateSlot(); + + dispatcher_tls->set( + [this](Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { + Thread::LockGuard guard(dispatcher_mutex_); + registered_dispatchers_.push_back(dispatcher); + return nullptr; + }); + serverReady(); server.run(); } diff --git a/test/integration/server.h b/test/integration/server.h index d62e3f7dd5f48..b2bd5c0c02731 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -509,6 +509,12 @@ class IntegrationTestServer : public Logger::Loggable, void useAdminInterfaceToQuit(bool use) { use_admin_interface_to_quit_ = use; } bool useAdminInterfaceToQuit() { return use_admin_interface_to_quit_; } + absl::optional> getWorkerDispatcher(uint32_t index) { + Thread::LockGuard guard(dispatcher_mutex_); + ASSERT(index < registered_dispatchers_.size()); + return registered_dispatchers_[index]; + } + protected: IntegrationTestServer(Event::TestTimeSystem& time_system, Api::Api& api, const std::string& config_path) @@ -531,6 +537,10 @@ class IntegrationTestServer : public Logger::Loggable, // adminAddress()) will be available. void serverReady(); + Thread::MutexBasicLockable dispatcher_mutex_; + // Assume 128 is greater than concurrency. + std::vector>> registered_dispatchers_; + private: /** * Runs the real server on a thread. diff --git a/test/integration/socket_interface_integration_test.cc b/test/integration/socket_interface_integration_test.cc index 235867e577399..bfcf06fc50e7f 100644 --- a/test/integration/socket_interface_integration_test.cc +++ b/test/integration/socket_interface_integration_test.cc @@ -90,7 +90,6 @@ TEST_P(SocketInterfaceIntegrationTest, AddressWithSocketInterface) { } // Test that connecting to internal address will crash. -// TODO(lambdai): Add internal connection implementation to enable the connection creation. TEST_P(SocketInterfaceIntegrationTest, InternalAddressWithSocketInterface) { BaseIntegrationTest::initialize(); @@ -101,14 +100,41 @@ TEST_P(SocketInterfaceIntegrationTest, InternalAddressWithSocketInterface) { Network::Address::InstanceConstSharedPtr address = std::make_shared("listener_0", sock_interface); + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.internal_address", "false"}}); ASSERT_DEATH(client_ = dispatcher_->createClientConnection( address, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr), "panic: not implemented"); } +// Test that connecting to internal address will crash. +TEST_P(SocketInterfaceIntegrationTest, CreateClientConnectionToInternalAddress) { + + BaseIntegrationTest::initialize(); + + ConnectionStatusCallbacks connect_callbacks_; + Network::ClientConnectionPtr client_; + const Network::SocketInterface* sock_interface = Network::socketInterface( + "envoy.extensions.network.socket_interface.default_socket_interface"); + Network::Address::InstanceConstSharedPtr address = + std::make_shared("listener_0", sock_interface); + + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.internal_address", "true"}}); + client_ = dispatcher_->createClientConnection(address, Network::Address::InstanceConstSharedPtr(), + Network::Test::createRawBufferSocket(), nullptr); + client_->addConnectionCallbacks(connect_callbacks_); + client_->connect(); + while (!connect_callbacks_.connected() && !connect_callbacks_.closed()) { + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + ASSERT_TRUE(connect_callbacks_.closed()); + client_->close(Network::ConnectionCloseType::NoFlush); +} + // Test that recv from internal address will crash. -// TODO(lambdai): Add internal socket implementation to enable the io path. +// TODO(lambdai): Add UDP internal listener implementation to enable the io path. TEST_P(SocketInterfaceIntegrationTest, UdpRecvFromInternalAddressWithSocketInterface) { BaseIntegrationTest::initialize(); diff --git a/test/integration/tcp_proxy_integration_test.h b/test/integration/tcp_proxy_integration_test.h index 9bcaa758b774d..a458ec960dfa4 100644 --- a/test/integration/tcp_proxy_integration_test.h +++ b/test/integration/tcp_proxy_integration_test.h @@ -25,7 +25,6 @@ class TcpProxyIntegrationTest : public testing::TestWithParam overridden_listener, Network::ListenerConfig& config, AddListenerCompletion completion) -> void { UNREFERENCED_PARAMETER(overridden_listener); - config.listenSocketFactory().getListenSocket(); + if (!config.internalListenerConfig().has_value()) { + config.listenSocketFactory().getListenSocket(); + } EXPECT_EQ(nullptr, add_listener_completion_); add_listener_completion_ = completion; })); diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index b0b6f6a11f3e4..4e060bf500a30 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -9,7 +9,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/common/common:96.3" "source/common/common/posix:94.1" "source/common/crypto:0.0" -"source/common/event:94.2" # Emulated edge events guards don't report LCOV +"source/common/event:93.9" # Emulated edge events guards don't report LCOV "source/common/filesystem/posix:96.2" "source/common/json:90.9" "source/common/network:95.0" # Flaky, `activateFileEvents`, `startSecureTransport` and `ioctl` do not always report LCOV diff --git a/test/server/BUILD b/test/server/BUILD index 498bc7ecef0bb..d47f31b711eb4 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -74,19 +74,51 @@ envoy_cc_test( name = "connection_handler_test", srcs = ["connection_handler_test.cc"], deps = [ + ":utility_lib", + "//source/common/api:os_sys_calls_lib", "//source/common/common:utility_lib", + "//source/common/config:metadata_lib", "//source/common/config:utility_lib", + "//source/common/init:manager_lib", + "//source/common/network:addr_family_aware_socket_option_lib", "//source/common/network:address_lib", "//source/common/network:connection_balancer_lib", + "//source/common/network:listen_socket_lib", + "//source/common/network:socket_option_lib", "//source/common/network:udp_packet_writer_handler_lib", + "//source/common/network:utility_lib", + "//source/common/protobuf", "//source/common/stats:stats_lib", + "//source/extensions/filters/listener/original_dst:config", + "//source/extensions/filters/listener/proxy_protocol:config", + "//source/extensions/filters/listener/tls_inspector:config", + "//source/extensions/filters/network/http_connection_manager:config", + "//source/extensions/filters/network/tcp_proxy:config", + "//source/extensions/request_id/uuid:config", + "//source/extensions/transport_sockets/raw_buffer:config", + "//source/extensions/transport_sockets/tls:config", + "//source/extensions/transport_sockets/tls:ssl_socket_lib", "//source/server:active_raw_udp_listener_config", "//source/server:connection_handler_lib", + "//source/server:listener_manager_lib", "//test/mocks/access_log:access_log_mocks", "//test/mocks/api:api_mocks", + "//test/mocks/init:init_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/server:drain_manager_mocks", + "//test/mocks/server:guard_dog_mocks", + "//test/mocks/server:instance_mocks", + "//test/mocks/server:listener_component_factory_mocks", + "//test/mocks/server:worker_factory_mocks", + "//test/mocks/server:worker_mocks", + "//test/test_common:environment_lib", "//test/test_common:network_utility_lib", + "//test/test_common:registry_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", + "//test/test_common:test_time_lib", "//test/test_common:threadsafe_singleton_injector_lib", + "@envoy_api//envoy/admin/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", ], @@ -111,6 +143,57 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "active_internal_listener_test", + srcs = ["active_internal_listener_test.cc"], + deps = [ + ":utility_lib", + "//source/common/api:os_sys_calls_lib", + "//source/common/common:utility_lib", + "//source/common/config:metadata_lib", + "//source/common/config:utility_lib", + "//source/common/init:manager_lib", + "//source/common/network:addr_family_aware_socket_option_lib", + "//source/common/network:address_lib", + "//source/common/network:connection_balancer_lib", + "//source/common/network:listen_socket_lib", + "//source/common/network:socket_option_lib", + "//source/common/network:utility_lib", + "//source/common/protobuf", + "//source/common/stats:stats_lib", + "//source/extensions/filters/listener/original_dst:config", + "//source/extensions/filters/listener/proxy_protocol:config", + "//source/extensions/filters/listener/tls_inspector:config", + "//source/extensions/filters/network/http_connection_manager:config", + "//source/extensions/filters/network/tcp_proxy:config", + "//source/extensions/request_id/uuid:config", + "//source/extensions/transport_sockets/raw_buffer:config", + "//source/extensions/transport_sockets/tls:config", + "//source/extensions/transport_sockets/tls:ssl_socket_lib", + "//source/server:active_raw_udp_listener_config", + "//source/server:connection_handler_lib", + "//source/server:listener_manager_lib", + "//test/mocks/access_log:access_log_mocks", + "//test/mocks/api:api_mocks", + "//test/mocks/init:init_mocks", + "//test/mocks/network:network_mocks", + "//test/mocks/server:drain_manager_mocks", + "//test/mocks/server:guard_dog_mocks", + "//test/mocks/server:instance_mocks", + "//test/mocks/server:listener_component_factory_mocks", + "//test/mocks/server:worker_factory_mocks", + "//test/mocks/server:worker_mocks", + "//test/test_common:environment_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:registry_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", + "//test/test_common:test_time_lib", + "//test/test_common:threadsafe_singleton_injector_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "drain_manager_impl_test", srcs = ["drain_manager_impl_test.cc"], diff --git a/test/server/active_internal_listener_test.cc b/test/server/active_internal_listener_test.cc new file mode 100644 index 0000000000000..19b7f209d76e3 --- /dev/null +++ b/test/server/active_internal_listener_test.cc @@ -0,0 +1,188 @@ +#include + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/network/exception.h" +#include "envoy/network/filter.h" +#include "envoy/network/listener.h" +#include "envoy/stats/scope.h" + +#include "common/common/utility.h" +#include "common/config/utility.h" +#include "common/network/address_impl.h" +#include "common/network/connection_balancer_impl.h" +#include "common/network/io_socket_handle_impl.h" +#include "common/network/raw_buffer_socket.h" +#include "common/network/utility.h" + +#include "server/active_internal_listener.h" +#include "server/connection_handler_impl.h" + +#include "test/mocks/access_log/mocks.h" +#include "test/mocks/api/mocks.h" +#include "test/mocks/common.h" +#include "test/mocks/network/mocks.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/threadsafe_singleton_injector.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Invoke; +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Server { +namespace { + +class MockInternalListenerCallback : public Network::InternalListenerCallbacks { +public: + MOCK_METHOD(void, onAccept, (Network::ConnectionSocketPtr && socket), ()); + MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); +}; +class ActiveInternalListenerTest : public testing::Test, + protected Logger::Loggable { +public: + ActiveInternalListenerTest() { + EXPECT_CALL(listener_config_, listenerScope).Times(testing::AnyNumber()); + EXPECT_CALL(conn_handler_, statPrefix()).WillRepeatedly(ReturnRef(listener_stat_prefix_)); + listener_filter_matcher_ = std::make_shared>(); + } + void addListener() { + EXPECT_CALL(listener_config_, listenerFiltersTimeout()); + EXPECT_CALL(listener_config_, continueOnListenerFiltersTimeout()); + EXPECT_CALL(listener_config_, filterChainManager()).WillRepeatedly(ReturnRef(manager_)); + EXPECT_CALL(listener_config_, openConnections()).WillRepeatedly(ReturnRef(resource_limit_)); + auto mock_listener_will_be_moved = std::make_unique(); + generic_listener_ = mock_listener_will_be_moved.get(); + internal_listener_ = std::make_shared( + conn_handler_, dispatcher_, std::move(mock_listener_will_be_moved), listener_config_); + } + Network::Listener* addListenerWithRealNetworkListener() { + EXPECT_CALL(listener_config_, listenerFiltersTimeout()); + EXPECT_CALL(listener_config_, continueOnListenerFiltersTimeout()); + EXPECT_CALL(listener_config_, filterChainManager()).WillRepeatedly(ReturnRef(manager_)); + EXPECT_CALL(listener_config_, openConnections()).WillRepeatedly(ReturnRef(resource_limit_)); + + internal_listener_ = + std::make_shared(conn_handler_, dispatcher_, listener_config_); + return internal_listener_->listener(); + } + void expectFilterChainFactory() { + EXPECT_CALL(listener_config_, filterChainFactory()) + .WillRepeatedly(ReturnRef(filter_chain_factory_)); + } + std::string listener_stat_prefix_{"listener_stat_prefix"}; + NiceMock dispatcher_{"test"}; + BasicResourceLimitImpl resource_limit_; + Network::MockConnectionHandler conn_handler_; + Network::MockListener* generic_listener_; + Network::MockListenerConfig listener_config_; + NiceMock manager_; + NiceMock filter_chain_factory_; + std::shared_ptr filter_chain_; + std::shared_ptr> listener_filter_matcher_; + std::shared_ptr internal_listener_; +}; + +TEST_F(ActiveInternalListenerTest, BasicInternalListener) { + addListener(); + EXPECT_CALL(*generic_listener_, onDestroy()); +} + +TEST_F(ActiveInternalListenerTest, AcceptSocketAndCreateListenerFilter) { + addListener(); + expectFilterChainFactory(); + Network::MockListenerFilter* test_listener_filter = new Network::MockListenerFilter(); + // FIX-ME: replace by mock socket + Network::Address::InstanceConstSharedPtr original_dst_address( + new Network::Address::Ipv4Instance("127.0.0.2", 8080)); + + Network::MockConnectionSocket* accepted_socket = new NiceMock(); + + EXPECT_CALL(filter_chain_factory_, createListenerFilterChain(_)) + .WillRepeatedly(Invoke([&](Network::ListenerFilterManager& manager) -> bool { + // Insert the Mock filter. + manager.addAcceptFilter(listener_filter_matcher_, + Network::ListenerFilterPtr{test_listener_filter}); + return true; + })); + EXPECT_CALL(*test_listener_filter, onAccept(_)) + .WillOnce(Invoke([&](Network::ListenerFilterCallbacks& cb) -> Network::FilterStatus { + cb.socket().addressProvider().restoreLocalAddress(original_dst_address); + return Network::FilterStatus::Continue; + })); + EXPECT_CALL(*test_listener_filter, destroy_()); + EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(nullptr)); + internal_listener_->onAccept(Network::ConnectionSocketPtr{accepted_socket}); + EXPECT_CALL(*generic_listener_, onDestroy()); +} + +TEST_F(ActiveInternalListenerTest, AcceptSocketAndCreateNetworkFilter) { + + addListener(); + expectFilterChainFactory(); + + Network::MockListenerFilter* test_listener_filter = new Network::MockListenerFilter(); + // FIX-ME: replace by mock socket + Network::Address::InstanceConstSharedPtr original_dst_address( + new Network::Address::Ipv4Instance("127.0.0.2", 8080)); + + Network::MockConnectionSocket* accepted_socket = new NiceMock(); + + EXPECT_CALL(filter_chain_factory_, createListenerFilterChain(_)) + .WillRepeatedly(Invoke([&](Network::ListenerFilterManager& manager) -> bool { + // Insert the Mock filter. + manager.addAcceptFilter(listener_filter_matcher_, + Network::ListenerFilterPtr{test_listener_filter}); + return true; + })); + EXPECT_CALL(*test_listener_filter, onAccept(_)) + .WillOnce(Invoke([&](Network::ListenerFilterCallbacks& cb) -> Network::FilterStatus { + cb.socket().addressProvider().restoreLocalAddress(original_dst_address); + return Network::FilterStatus::Continue; + })); + EXPECT_CALL(*test_listener_filter, destroy_()); + auto filter_factory_callback = std::make_shared>(); + filter_chain_ = std::make_shared>(); + auto transport_socket_factory = Network::Test::createRawBufferSocketFactory(); + + EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(filter_chain_.get())); + EXPECT_CALL(*filter_chain_, transportSocketFactory) + .WillOnce(testing::ReturnRef(*transport_socket_factory)); + EXPECT_CALL(*filter_chain_, networkFilterFactories).WillOnce(ReturnRef(*filter_factory_callback)); + auto* connection = new NiceMock(); + EXPECT_CALL(dispatcher_, createServerConnection_()).WillOnce(Return(connection)); + EXPECT_CALL(conn_handler_, incNumConnections()); + EXPECT_CALL(filter_chain_factory_, createNetworkFilterChain(_, _)).WillOnce(Return(true)); + EXPECT_CALL(listener_config_, perConnectionBufferLimitBytes()); + internal_listener_->onAccept(Network::ConnectionSocketPtr{accepted_socket}); + EXPECT_CALL(conn_handler_, decNumConnections()); + connection->close(Network::ConnectionCloseType::NoFlush); + dispatcher_.clearDeferredDeleteList(); + EXPECT_CALL(*generic_listener_, onDestroy()); +} + +TEST_F(ActiveInternalListenerTest, StopListener) { + addListener(); + EXPECT_CALL(*generic_listener_, onDestroy()); + internal_listener_->shutdownListener(); +} + +TEST_F(ActiveInternalListenerTest, PausedListenerAcceptNewSocket) { + addListenerWithRealNetworkListener(); + internal_listener_->pauseListening(); + + expectFilterChainFactory(); + Network::MockConnectionSocket* accepted_socket = new NiceMock(); + + EXPECT_CALL(filter_chain_factory_, createListenerFilterChain(_)) + .WillRepeatedly(Invoke([&](Network::ListenerFilterManager&) -> bool { return true; })); + EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(nullptr)); + internal_listener_->onAccept(Network::ConnectionSocketPtr{accepted_socket}); +} +} // namespace +} // namespace Server +} // namespace Envoy diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index c15f984a4e943..6dbc613ba66eb 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -1,3 +1,53 @@ +#include +#include +#include +#include +#include + +#include "envoy/admin/v3/config_dump.pb.h" +#include "envoy/config/core/v3/address.pb.h" +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/listener/v3/listener.pb.h" +#include "envoy/network/listener.h" +#include "envoy/server/filter_config.h" +#include "envoy/server/listener_manager.h" +#include "envoy/stream_info/filter_state.h" + +#include "common/api/os_sys_calls_impl.h" +#include "common/common/macros.h" +#include "common/config/metadata.h" +#include "common/init/manager_impl.h" +#include "common/network/address_impl.h" +#include "common/network/io_socket_handle_impl.h" +#include "common/network/utility.h" +#include "common/protobuf/protobuf.h" + +#include "server/configuration_impl.h" +#include "server/listener_manager_impl.h" + +#include "test/mocks/init/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/mocks/server/drain_manager.h" +#include "test/mocks/server/guard_dog.h" +#include "test/mocks/server/instance.h" +#include "test/mocks/server/listener_component_factory.h" +#include "test/mocks/server/worker.h" +#include "test/mocks/server/worker_factory.h" +#include "test/server/utility.h" +#include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/registry.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" +#include "test/test_common/threadsafe_singleton_injector.h" +#include "test/test_common/utility.h" + +#include "absl/strings/escaping.h" +#include "absl/strings/match.h" + +// Above from listener_manager_impl_test.cc + #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/listener/v3/udp_listener_config.pb.h" #include "envoy/network/exception.h" @@ -107,6 +157,8 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable udp_listener_config_; + std::unique_ptr internal_listener_config_; Network::ConnectionBalancerSharedPtr connection_balancer_; BasicResourceLimitImpl open_connections_; const std::vector access_logs_; @@ -255,6 +315,22 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable> overridden_filter_chain_manager = + nullptr) { + listeners_.emplace_back(std::make_unique( + *this, tag, /*bind_to_port*/ false, /*hand_off_restored_destination_connections*/ false, + name, Network::Socket::Type::Stream, listener_filters_timeout, + continue_on_listener_filters_timeout, socket_factory_, access_log_, + overridden_filter_chain_manager, ENVOY_TCP_BACKLOG_SIZE, nullptr)); + listeners_.back()->internal_listener_config_ = + std::make_unique(); + return listeners_.back().get(); + } + void validateOriginalDst(Network::TcpListenerCallbacks** listener_callbacks, TestListener* test_listener, Network::MockListener* listener) { Network::Address::InstanceConstSharedPtr normal_address( @@ -299,7 +375,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable dispatcher_{"test"}; std::list listeners_; - Network::ConnectionHandlerPtr handler_; + std::unique_ptr handler_; NiceMock manager_; NiceMock factory_; const std::shared_ptr filter_chain_; @@ -1368,6 +1444,67 @@ TEST_F(ConnectionHandlerTest, TcpBacklogCustom) { handler_->addListener(absl::nullopt, *test_listener); } +TEST_F(ConnectionHandlerTest, DisableInternalListener) { + InSequence s; + Network::Address::InstanceConstSharedPtr local_address{ + new Network::Address::EnvoyInternalInstance("server_internal_address")}; + + TestListener* internal_listener = + addInternalListener(1, "test_internal_listener", std::chrono::milliseconds(), false, nullptr); + EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address)); + handler_->addListener(absl::nullopt, *internal_listener); + auto internal_listener_cb = handler_->findByAddress(local_address); + ASSERT(internal_listener_cb.has_value()); + + handler_->disableListeners(); + auto internal_listener_cb_disabled = handler_->findByAddress(local_address); + ASSERT(internal_listener_cb_disabled.has_value()); + ASSERT_EQ(&internal_listener_cb_disabled.value().get(), &internal_listener_cb.value().get()); + + handler_->enableListeners(); + auto internal_listener_cb_enabled = handler_->findByAddress(local_address); + ASSERT(internal_listener_cb_enabled.has_value()); + ASSERT_EQ(&internal_listener_cb_enabled.value().get(), &internal_listener_cb.value().get()); +} + +TEST_F(ConnectionHandlerTest, InternalListenerInplaceUpdate) { + InSequence s; + uint64_t old_listener_tag = 1; + uint64_t new_listener_tag = 2; + Network::Address::InstanceConstSharedPtr local_address{ + new Network::Address::EnvoyInternalInstance("server_internal_address")}; + + TestListener* internal_listener = addInternalListener( + old_listener_tag, "test_internal_listener", std::chrono::milliseconds(), false, nullptr); + EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address)); + handler_->addListener(absl::nullopt, *internal_listener); + + ASSERT_NE(internal_listener, nullptr); + + auto overridden_filter_chain_manager = + std::make_shared>(); + TestListener* new_test_listener = + addInternalListener(new_listener_tag, "test_internal_listener", std::chrono::milliseconds(), + false, overridden_filter_chain_manager); + + handler_->addListener(old_listener_tag, *new_test_listener); + + Network::MockConnectionSocket* connection = new NiceMock(); + + auto internal_listener_cb = handler_->findByAddress(local_address); + + EXPECT_CALL(manager_, findFilterChain(_)).Times(0); + EXPECT_CALL(*overridden_filter_chain_manager, findFilterChain(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(*access_log_, log(_, _, _, _)); + internal_listener_cb.value().get().onAccept(Network::ConnectionSocketPtr{connection}); + EXPECT_EQ(0UL, handler_->numConnections()); + + testing::MockFunction completion; + handler_->removeFilterChains(old_listener_tag, {}, completion.AsStdFunction()); + EXPECT_CALL(completion, Call()); + dispatcher_.clearDeferredDeleteList(); +} + } // namespace } // namespace Server } // namespace Envoy diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 246579d4f1295..23ff4cc075a51 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -581,6 +581,10 @@ bind_to_port: false } TEST_F(ListenerManagerImplTest, UnsupportedInternalListener) { + auto scoped_runtime_guard = std::make_unique(); + + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.internal_address", "false"}}); const std::string yaml = R"EOF( address: envoy_internal_address: @@ -4725,6 +4729,313 @@ name: test_api_listener_2 EXPECT_EQ("test_api_listener", manager_->apiListener()->get().name()); } +TEST_F(ListenerManagerImplWithRealFiltersTest, AddOrUpdateInternalListener) { + auto scoped_runtime_guard = std::make_unique(); + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.internal_address", "true"}}); + time_system_.setSystemTime(std::chrono::milliseconds(1001001001001)); + + InSequence s; + + auto* lds_api = new MockLdsApi(); + EXPECT_CALL(listener_factory_, createLdsApi_(_, _)).WillOnce(Return(lds_api)); + envoy::config::core::v3::ConfigSource lds_config; + manager_->createLdsApi(lds_config, nullptr); + + EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("")); + checkConfigDump(R"EOF( +static_listeners: +)EOF"); + + // Add foo listener. + const std::string listener_foo_yaml = R"EOF( +name: test_internal_listener +address: + envoy_internal_address: + server_listener_name: test_internal_listener_name +internal_listener: {} +filter_chains: {} + )EOF"; + + ListenerHandle* listener_foo = expectListenerCreate(false, true); + // EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); + EXPECT_TRUE( + manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml), "version1", true)); + checkStats(__LINE__, 1, 0, 0, 0, 1, 0, 0); + EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("version1")); + checkConfigDump(R"EOF( +version_info: version1 +static_listeners: +dynamic_listeners: + - name: test_internal_listener + warming_state: + version_info: version1 + listener: + "@type": type.googleapis.com/envoy.config.listener.v3.Listener + name: test_internal_listener + address: + envoy_internal_address: + server_listener_name: test_internal_listener_name + filter_chains: {} + internal_listener: {} + last_updated: + seconds: 1001001001 + nanos: 1000000 +)EOF"); + + // Update duplicate should be a NOP. + EXPECT_FALSE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml), "", true)); + checkStats(__LINE__, 1, 0, 0, 0, 1, 0, 0); + + // Update foo listener. Should share socket. + const std::string listener_foo_update1_yaml = R"EOF( +name: test_internal_listener +address: + envoy_internal_address: + server_listener_name: test_internal_listener_name +filter_chains: {} +internal_listener: {} +per_connection_buffer_limit_bytes: 10 + )EOF"; + + time_system_.setSystemTime(std::chrono::milliseconds(2002002002002)); + + ListenerHandle* listener_foo_update1 = expectListenerCreate(false, true); + EXPECT_CALL(*listener_foo, onDestroy()); + EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_update1_yaml), + "version2", true)); + checkStats(__LINE__, 1, 1, 0, 0, 1, 0, 0); + EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("version2")); + checkConfigDump(R"EOF( + version_info: version2 + static_listeners: + dynamic_listeners: + - name: test_internal_listener + warming_state: + version_info: version2 + listener: + "@type": type.googleapis.com/envoy.config.listener.v3.Listener + name: test_internal_listener + address: + envoy_internal_address: + server_listener_name: test_internal_listener_name + filter_chains: {} + internal_listener: {} + per_connection_buffer_limit_bytes: 10 + last_updated: + seconds: 2002002002 + nanos: 2000000 + )EOF"); + + // Validate that workers_started stat is zero before calling startWorkers. + EXPECT_EQ(0, server_.stats_store_ + .gauge("listener_manager.workers_started", Stats::Gauge::ImportMode::NeverImport) + .value()); + + // Start workers. + EXPECT_CALL(*worker_, addListener(_, _, _)); + EXPECT_CALL(*worker_, start(_)); + manager_->startWorkers(guard_dog_, callback_.AsStdFunction()); + // Validate that workers_started stat is still zero before workers set the status via + // completion callback. + EXPECT_EQ(0, server_.stats_store_ + .gauge("listener_manager.workers_started", Stats::Gauge::ImportMode::NeverImport) + .value()); + worker_->callAddCompletion(true); + + // Validate that workers_started stat is set to 1 after workers have responded with initialization + // status. + EXPECT_EQ(1, server_.stats_store_ + .gauge("listener_manager.workers_started", Stats::Gauge::ImportMode::NeverImport) + .value()); + + // Update duplicate should be a NOP. + EXPECT_FALSE( + manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_update1_yaml), "", true)); + checkStats(__LINE__, 1, 1, 0, 0, 1, 0, 0); + + time_system_.setSystemTime(std::chrono::milliseconds(3003003003003)); + + // Update foo. Should go into warming, have an immediate warming callback, and start immediate + // removal. + ListenerHandle* listener_foo_update2 = expectListenerCreate(false, true); + EXPECT_CALL(*worker_, addListener(_, _, _)); + EXPECT_CALL(*worker_, stopListener(_, _)); + EXPECT_CALL(*listener_foo_update1->drain_manager_, startDrainSequence(_)); + EXPECT_TRUE( + manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml), "version3", true)); + worker_->callAddCompletion(true); + checkStats(__LINE__, 1, 2, 0, 0, 1, 1, 0); + EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("version3")); + checkConfigDump(R"EOF( + version_info: version3 + static_listeners: + dynamic_listeners: + - name: test_internal_listener + active_state: + version_info: version3 + listener: + "@type": type.googleapis.com/envoy.config.listener.v3.Listener + name: test_internal_listener + address: + envoy_internal_address: + server_listener_name: test_internal_listener_name + filter_chains: {} + internal_listener: {} + last_updated: + seconds: 3003003003 + nanos: 3000000 + draining_state: + version_info: version2 + listener: + "@type": type.googleapis.com/envoy.config.listener.v3.Listener + name: test_internal_listener + address: + envoy_internal_address: + server_listener_name: test_internal_listener_name + filter_chains: {} + internal_listener: {} + per_connection_buffer_limit_bytes: 10 + last_updated: + seconds: 2002002002 + nanos: 2000000 + )EOF"); + + EXPECT_CALL(*worker_, removeListener(_, _)); + listener_foo_update1->drain_manager_->drain_sequence_completion_(); + checkStats(__LINE__, 1, 2, 0, 0, 1, 1, 0); + EXPECT_CALL(*listener_foo_update1, onDestroy()); + worker_->callRemovalCompletion(); + checkStats(__LINE__, 1, 2, 0, 0, 1, 0, 0); + + time_system_.setSystemTime(std::chrono::milliseconds(4004004004004)); + + // Add bar listener. + const std::string listener_bar_yaml = R"EOF( + name: test_internal_listener_bar + address: + envoy_internal_address: + server_listener_name: test_internal_listener_bar + filter_chains: {} + internal_listener: {} + )EOF"; + + ListenerHandle* listener_bar = expectListenerCreate(false, true); + // EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); + EXPECT_CALL(*worker_, addListener(_, _, _)); + EXPECT_TRUE( + manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_bar_yaml), "version4", true)); + EXPECT_EQ(2UL, manager_->listeners().size()); + worker_->callAddCompletion(true); + checkStats(__LINE__, 2, 2, 0, 0, 2, 0, 0); + + time_system_.setSystemTime(std::chrono::milliseconds(5005005005005)); + + // Add baz listener, this time requiring initializing. + const std::string listener_baz_yaml = R"EOF( + name: test_internal_listener_baz + address: + envoy_internal_address: + server_listener_name: test_internal_listener_baz + filter_chains: {} + internal_listener: {} + )EOF"; + + ListenerHandle* listener_baz = expectListenerCreate(true, true); + // EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); + EXPECT_CALL(listener_baz->target_, initialize()); + EXPECT_TRUE( + manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_baz_yaml), "version5", true)); + EXPECT_EQ(2UL, manager_->listeners().size()); + checkStats(__LINE__, 3, 2, 0, 1, 2, 0, 0); + EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("version5")); + checkConfigDump(R"EOF( + version_info: version5 + dynamic_listeners: + - name: test_internal_listener + active_state: + version_info: version3 + listener: + "@type": type.googleapis.com/envoy.config.listener.v3.Listener + name: test_internal_listener + address: + envoy_internal_address: + server_listener_name: test_internal_listener_name + filter_chains: {} + internal_listener: {} + last_updated: + seconds: 3003003003 + nanos: 3000000 + - name: test_internal_listener_bar + active_state: + version_info: version4 + listener: + "@type": type.googleapis.com/envoy.config.listener.v3.Listener + name: test_internal_listener_bar + address: + envoy_internal_address: + server_listener_name: test_internal_listener_bar + filter_chains: {} + internal_listener: {} + last_updated: + seconds: 4004004004 + nanos: 4000000 + - name: test_internal_listener_baz + warming_state: + version_info: version5 + listener: + "@type": type.googleapis.com/envoy.config.listener.v3.Listener + name: test_internal_listener_baz + address: + envoy_internal_address: + server_listener_name: test_internal_listener_baz + filter_chains: {} + internal_listener: {} + last_updated: + seconds: 5005005005 + nanos: 5000000 + )EOF"); + + // Update a duplicate baz that is currently warming. + EXPECT_FALSE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_baz_yaml), "", true)); + checkStats(__LINE__, 3, 2, 0, 1, 2, 0, 0); + + // Update baz while it is warming. + const std::string listener_baz_update1_yaml = R"EOF( + name: test_internal_listener_baz + address: + envoy_internal_address: + server_listener_name: test_internal_listener_baz + internal_listener: {} + filter_chains: + - filters: + - name: fake + typed_config: {} + )EOF"; + + ListenerHandle* listener_baz_update1 = expectListenerCreate(true, true); + EXPECT_CALL(*listener_baz, onDestroy()).WillOnce(Invoke([listener_baz]() -> void { + // Call the initialize callback during destruction like RDS will. + listener_baz->target_.ready(); + })); + EXPECT_CALL(listener_baz_update1->target_, initialize()); + EXPECT_TRUE( + manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_baz_update1_yaml), "", true)); + EXPECT_EQ(2UL, manager_->listeners().size()); + checkStats(__LINE__, 3, 3, 0, 1, 2, 0, 0); + + // Finish initialization for baz which should make it active. + EXPECT_CALL(*worker_, addListener(_, _, _)); + listener_baz_update1->target_.ready(); + EXPECT_EQ(3UL, manager_->listeners().size()); + worker_->callAddCompletion(true); + checkStats(__LINE__, 3, 3, 0, 0, 3, 0, 0); + + EXPECT_CALL(*listener_foo_update2, onDestroy()); + EXPECT_CALL(*listener_bar, onDestroy()); + EXPECT_CALL(*listener_baz_update1, onDestroy()); +} + TEST_F(ListenerManagerImplTest, StopInplaceWarmingListener) { InSequence s; From 4c156ce22d77d0df96485b5623e2235963d6ff8d Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 1 Jun 2021 14:29:56 -0700 Subject: [PATCH 02/23] for new pr Signed-off-by: Yuchen Dai --- include/envoy/event/dispatcher.h | 2 +- source/common/event/BUILD | 3 -- source/common/event/dispatcher_impl.cc | 45 +------------------ source/common/event/dispatcher_impl.h | 2 +- source/common/network/connection_impl.h | 1 - source/common/network/socket_impl.cc | 15 ++----- source/common/tcp_proxy/tcp_proxy.cc | 12 ++--- .../io_socket/user_space/io_handle_impl.cc | 5 +-- .../io_socket/user_space/io_handle_impl.h | 2 +- source/server/active_tcp_listener.cc | 3 +- source/server/active_tcp_listener.h | 3 +- source/server/listener_impl.cc | 2 - test/common/network/connection_impl_test.cc | 22 +++++---- test/integration/BUILD | 45 ------------------- test/integration/integration_tcp_client.cc | 45 ------------------- test/integration/integration_tcp_client.h | 5 --- .../internal_tcp_proxy_integration_test.cc | 2 +- .../socket_interface_integration_test.cc | 25 ----------- test/integration/tcp_proxy_integration_test.h | 1 + test/mocks/event/mocks.h | 6 +-- test/mocks/event/wrapped_dispatcher.h | 4 +- test/per_file_coverage.sh | 2 +- 22 files changed, 32 insertions(+), 220 deletions(-) diff --git a/include/envoy/event/dispatcher.h b/include/envoy/event/dispatcher.h index fcdbbcecef0f3..24b9f1a8bd502 100644 --- a/include/envoy/event/dispatcher.h +++ b/include/envoy/event/dispatcher.h @@ -208,7 +208,7 @@ class Dispatcher : public DispatcherBase { virtual void registerInternalListenerManager(Network::InternalListenerManager& internal_listener_manager) PURE; - virtual Network::InternalListenerManagerOptRef getInternalListenerManagerForTest() PURE; + virtual Network::InternalListenerManagerOptRef getInternalListenerManager() PURE; /** * Creates an async DNS resolver. The resolver should only be used on the thread that runs this diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 2c5f3b796b529..333755eeac351 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -46,9 +46,6 @@ envoy_cc_library( "//source/common/network:dns_lib", "//source/common/network:connection_lib", "//source/common/network:listener_lib", - - #internal address - "//source/extensions/io_socket/user_space:config", ] + select({ "//bazel:apple": ["//source/common/network:apple_dns_lib"], "//conditions:default": [], diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 07c615a08587c..60e624d2cf2cd 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -27,9 +27,6 @@ #include "common/network/udp_listener_impl.h" #include "common/runtime/runtime_features.h" -// internal address -#include "extensions/io_socket/user_space/io_handle_impl.h" - #include "event2/event.h" #ifdef ENVOY_HANDLE_SIGNALS @@ -153,47 +150,7 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr Network::Address::InstanceConstSharedPtr source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) { - FANCY_LOG(info, "lambdai: create client connection to {} from {}", - address != nullptr ? address->asStringView() : "nullptr", - source_address != nullptr ? source_address->asStringView() : "nullptr"); ASSERT(isThreadSafe()); - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.internal_address") && - address->type() == Network::Address::Type::EnvoyInternal) { - Network::IoHandlePtr io_handle_client; - Network::IoHandlePtr io_handle_server; - - std::tie(io_handle_client, io_handle_server) = - Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); - - auto client_conn = std::make_unique( - *this, std::move(io_handle_client), address, source_address, std::move(transport_socket), - options); - // TODO(lambdai): refactor nested if. - auto internal_listener_manager = getInternalListenerManagerForTest(); - if (internal_listener_manager.has_value()) { - // It's either in main thread or the worker is not yet started. - auto internal_listener = internal_listener_manager.value().get().findByAddress(address); - if (internal_listener.has_value()) { - auto accepted_socket = std::make_unique( - std::move(io_handle_server), address, source_address); - // TODO: also check if disabled - internal_listener.value().get().onAccept(std::move(accepted_socket)); - FANCY_LOG(info, "lambdai: find internal listener {} ", address->asStringView()); - } else { - FANCY_LOG(info, "lambdai: cannot find internal listener {} from internal listener manager", - address->asStringView()); - // injected error into client_conn; - io_handle_server->close(); - } - } else { - FANCY_LOG(info, "lambdai: cannot find from internal listener manager while connecting to {}", - address->asStringView()); - // injected error into client_conn; - io_handle_server->close(); - } - - return client_conn; - } return std::make_unique(*this, address, source_address, std::move(transport_socket), options); } @@ -263,7 +220,7 @@ void DispatcherImpl::registerInternalListenerManager( internal_listener_manager_ = internal_listener_manager; } -Network::InternalListenerManagerOptRef DispatcherImpl::getInternalListenerManagerForTest() { +Network::InternalListenerManagerOptRef DispatcherImpl::getInternalListenerManager() { return internal_listener_manager_; } diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index 3add83447e3f2..4b2965da4bcd4 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -81,7 +81,7 @@ class DispatcherImpl : Logger::Loggable, void registerInternalListenerManager( Network::InternalListenerManager& internal_listener_manager) override; - Network::InternalListenerManagerOptRef getInternalListenerManagerForTest() override; + Network::InternalListenerManagerOptRef getInternalListenerManager() override; TimerPtr createTimer(TimerCb cb) override; TimerPtr createScaledTimer(ScaledTimerType timer_type, TimerCb cb) override; diff --git a/source/common/network/connection_impl.h b/source/common/network/connection_impl.h index 90eda7f5397d7..1c08493226054 100644 --- a/source/common/network/connection_impl.h +++ b/source/common/network/connection_impl.h @@ -12,7 +12,6 @@ #include "common/buffer/watermark_buffer.h" #include "common/event/libevent.h" #include "common/network/connection_impl_base.h" -#include "common/network/listen_socket_impl.h" #include "common/stream_info/stream_info_impl.h" #include "absl/types/optional.h" diff --git a/source/common/network/socket_impl.cc b/source/common/network/socket_impl.cc index e0c81f5cc39bc..d7fb6d659c9f1 100644 --- a/source/common/network/socket_impl.cc +++ b/source/common/network/socket_impl.cc @@ -22,19 +22,10 @@ SocketImpl::SocketImpl(IoHandlePtr&& io_handle, : io_handle_(std::move(io_handle)), address_provider_(std::make_shared(local_address, remote_address)) { - FANCY_LOG(debug, - "lambdai: SocketImpl::SocketImpl( local address = {} provider local address = {}", - local_address == nullptr ? "nullptr" : local_address->asStringView(), - address_provider_->localAddress() == nullptr - ? "nullptr" - : address_provider_->localAddress()->asStringView()); - FANCY_LOG(debug, - "lambdai: SocketImpl::SocketImpl( remote address = {} provider local address = {}", - remote_address == nullptr ? "nullptr" : remote_address->asStringView(), - address_provider_->remoteAddress() == nullptr - ? "nullptr" - : address_provider_->remoteAddress()->asStringView()); if (address_provider_->localAddress() != nullptr) { + // The remote address should have the exact local address type. + ASSERT(address_provider_->remoteAddress() == nullptr || + address_provider_->remoteAddress()->type() == address_provider_->localAddress()->type()); addr_type_ = address_provider_->localAddress()->type(); return; } diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index d0a0fc7fa4a74..2c7cff6f6008e 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -589,8 +589,8 @@ Network::FilterStatus Filter::onNewConnection() { } void Filter::onDownstreamEvent(Network::ConnectionEvent event) { - ENVOY_LOG(debug, "lambdai: tcp proxy filter on downstream event {}, upstream_ = {}", - static_cast(event), static_cast(upstream_.get())); + ENVOY_CONN_LOG(trace, "on downstream event {}, has upstream = {}", read_callbacks_->connection(), + static_cast(event), upstream_ == nullptr); if (upstream_) { Tcp::ConnectionPool::ConnectionDataPtr conn_data(upstream_->onDownstreamEvent(event)); if (conn_data != nullptr && @@ -751,11 +751,9 @@ Drainer::Drainer(UpstreamDrainManager& parent, const Config::SharedConfigSharedP const Upstream::HostDescriptionConstSharedPtr& upstream_host) : parent_(parent), callbacks_(callbacks), upstream_conn_data_(std::move(conn_data)), timer_(std::move(idle_timer)), upstream_host_(upstream_host), config_(config) { + ENVOY_CONN_LOG(debug, "draining the upstream connection", upstream_conn_data_->connection()); config_->stats().upstream_flush_total_.inc(); config_->stats().upstream_flush_active_.inc(); - ENVOY_CONN_LOG(debug, "set drainer and bump upstream_flush_total_ to {}, active to {} ", - upstream_conn_data_->connection(), config_->stats().upstream_flush_total_.value(), - config_->stats().upstream_flush_active_.value()); } void Drainer::onEvent(Network::ConnectionEvent event) { @@ -765,10 +763,6 @@ void Drainer::onEvent(Network::ConnectionEvent event) { timer_->disableTimer(); } config_->stats().upstream_flush_active_.dec(); - ENVOY_CONN_LOG( - debug, "drainer on close event and set upstream_flush_total_ to {}, active to {}", - upstream_conn_data_->connection(), config_->stats().upstream_flush_total_.value(), - config_->stats().upstream_flush_active_.value()); parent_.remove(*this, upstream_conn_data_->connection().dispatcher()); } } diff --git a/source/extensions/io_socket/user_space/io_handle_impl.cc b/source/extensions/io_socket/user_space/io_handle_impl.cc index 495aa8bf10ac2..9a46d7fb788fd 100644 --- a/source/extensions/io_socket/user_space/io_handle_impl.cc +++ b/source/extensions/io_socket/user_space/io_handle_impl.cc @@ -277,14 +277,11 @@ Network::IoHandlePtr IoHandleImpl::accept(struct sockaddr*, socklen_t*) { Api::SysCallIntResult IoHandleImpl::connect(Network::Address::InstanceConstSharedPtr address) { if (peer_handle_ != nullptr) { - FANCY_LOG(info, "lambdai: user handle {} connect to {} has peer_handler, returning 0", - static_cast(this), address->asStringView()); - // Buffered Io handle should always be considered as connected unless the server peer cannot be // found. Use write or read to determine if peer is closed. return {0, 0}; } else { - FANCY_LOG(info, "lambdai: user handle {} connect to {}. no peer_handler, returning error", + ENVOY_LOG(debug, "user namespace handle {} connect to {} the closed peer.", static_cast(this), address->asStringView()); return Api::SysCallIntResult{-1, SOCKET_ERROR_INVAL}; } diff --git a/source/extensions/io_socket/user_space/io_handle_impl.h b/source/extensions/io_socket/user_space/io_handle_impl.h index d9feb7991d8ed..19f8bba69fa77 100644 --- a/source/extensions/io_socket/user_space/io_handle_impl.h +++ b/source/extensions/io_socket/user_space/io_handle_impl.h @@ -144,7 +144,7 @@ class IoHandleImpl final : public Network::IoHandle, ASSERT(!peer_handle_); ASSERT(!write_shutdown_); peer_handle_ = writable_peer; - FANCY_LOG(debug, "lambdai: set iohandle {} set peer_handle_ to {} ", static_cast(this), + ENVOY_LOG(trace, "io handle {} set peer handle to {}.", static_cast(this), static_cast(writable_peer)); } diff --git a/source/server/active_tcp_listener.cc b/source/server/active_tcp_listener.cc index 54d7a82e9892e..94f4ec4ddd0e6 100644 --- a/source/server/active_tcp_listener.cc +++ b/source/server/active_tcp_listener.cc @@ -256,8 +256,7 @@ ActiveTcpConnection::~ActiveTcpConnection() { // Network::ConnectionCallbacks void ActiveTcpConnection::onEvent(Network::ConnectionEvent event) { - FANCY_LOG(info, "lambdai: tcp connection on event {}", static_cast(event)); - + ENVOY_LOG(trace, "[C{}] connection on event {}", connection_->id(), static_cast(event)); // Any event leads to destruction of the connection. if (event == Network::ConnectionEvent::LocalClose || event == Network::ConnectionEvent::RemoteClose) { diff --git a/source/server/active_tcp_listener.h b/source/server/active_tcp_listener.h index 09a282f077a40..02b1f3d4e4f69 100644 --- a/source/server/active_tcp_listener.h +++ b/source/server/active_tcp_listener.h @@ -48,7 +48,8 @@ class ActiveConnections : public Event::DeferredDeletable { */ struct ActiveTcpConnection : LinkedObject, public Event::DeferredDeletable, - public Network::ConnectionCallbacks { + public Network::ConnectionCallbacks, + Logger::Loggable { ActiveTcpConnection(ActiveConnections& active_connections, Network::ConnectionPtr&& new_connection, TimeSource& time_system, std::unique_ptr&& stream_info); diff --git a/source/server/listener_impl.cc b/source/server/listener_impl.cc index 35541132b89d1..0cf43c455ca73 100644 --- a/source/server/listener_impl.cc +++ b/source/server/listener_impl.cc @@ -1,7 +1,5 @@ #include "server/listener_impl.h" -#include - #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/listener/v3/listener.pb.h" #include "envoy/config/listener/v3/listener_components.pb.h" diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 370f5c55fd61b..832cae04eb75f 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -2968,8 +2968,8 @@ class InternalClientConnectionImplTest : public testing::Test { StrictMock client_callbacks_; }; -// Validate we skip setting source address. -TEST_F(InternalClientConnectionImplTest, SkipSourceAddress) { +TEST_F(InternalClientConnectionImplTest, + CannotCreateConnectionToInternalAddressWithInternalAddressEnabled) { auto scoped_runtime_guard = std::make_unique(); Runtime::LoaderSingleton::getExisting()->mergeValues( {{"envoy.reloadable_features.internal_address", "true"}}); @@ -2981,16 +2981,14 @@ TEST_F(InternalClientConnectionImplTest, SkipSourceAddress) { MockInternalListenerManager internal_listener_manager; dispatcher_->registerInternalListenerManager(internal_listener_manager); - EXPECT_CALL(internal_listener_manager, findByAddress(address)).WillOnce(Return(absl::nullopt)); - ClientConnectionPtr connection = - dispatcher_->createClientConnection(address, Network::Address::InstanceConstSharedPtr(), - Network::Test::createRawBufferSocket(), nullptr); - connection->addConnectionCallbacks(client_callbacks_); - connection->connect(); - EXPECT_CALL(client_callbacks_, onEvent(ConnectionEvent::RemoteClose)) - .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { dispatcher_->exit(); })); - dispatcher_->run(Event::Dispatcher::RunType::Block); - connection->close(ConnectionCloseType::NoFlush); + // Not implemented yet. + ASSERT_DEATH( + { + ClientConnectionPtr connection = + dispatcher_->createClientConnection(address, Network::Address::InstanceConstSharedPtr(), + Network::Test::createRawBufferSocket(), nullptr); + }, + "panic: not implemented"); } } // namespace diff --git a/test/integration/BUILD b/test/integration/BUILD index 8705c6c500c33..f7fd6a457487b 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -634,9 +634,6 @@ envoy_cc_test_library( "//test/mocks/buffer:buffer_mocks", "//test/test_common:network_utility_lib", "//test/test_common:utility_lib", - - # for internal - "//source/extensions/io_socket/user_space:config", ], ) @@ -1243,48 +1240,6 @@ envoy_cc_test( "@envoy_api//envoy/config/filter/network/tcp_proxy/v2:pkg_cc_proto", "@envoy_api//envoy/extensions/access_loggers/file/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto", - - # for internal - "//source/extensions/io_socket/user_space:config", - ], -) - -envoy_cc_test( - name = "internal_tcp_proxy_integration_test", - srcs = [ - "internal_tcp_proxy_integration_test.cc", - "tcp_proxy_integration_test.h", - ], - data = [ - "//test/config/integration/certs", - ], - shard_count = 1, - deps = [ - ":integration_lib", - ":tcp_proxy_integration_proto_cc_proto", - "//source/common/config:api_version_lib", - "//source/common/event:dispatcher_includes", - "//source/common/event:dispatcher_lib", - "//source/common/network:utility_lib", - "//source/extensions/access_loggers/file:config", - "//source/extensions/filters/network/common:factory_base_lib", - "//source/extensions/filters/network/tcp_proxy:config", - "//source/extensions/transport_sockets/tls:config", - "//source/extensions/transport_sockets/tls:context_config_lib", - "//source/extensions/transport_sockets/tls:context_lib", - "//test/mocks/runtime:runtime_mocks", - "//test/mocks/secret:secret_mocks", - "//test/test_common:registry_lib", - "//test/test_common:utility_lib", - "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/config/filter/network/tcp_proxy/v2:pkg_cc_proto", - "@envoy_api//envoy/extensions/access_loggers/file/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto", - - # for internal - "//source/extensions/io_socket/user_space:config", ], ) diff --git a/test/integration/integration_tcp_client.cc b/test/integration/integration_tcp_client.cc index f6356723eebd6..60d936ab0d2d3 100644 --- a/test/integration/integration_tcp_client.cc +++ b/test/integration/integration_tcp_client.cc @@ -16,13 +16,6 @@ #include "common/common/fmt.h" #include "common/network/utility.h" -/* begin internal connection */ -#include "common/stream_info/stream_info_impl.h" -#include "common/network/listen_socket_impl.h" -#include "common/network/connection_impl.h" -#include "extensions/io_socket/user_space/io_handle_impl.h" -/* end internal connection */ - #include "test/integration/utility.h" #include "test/mocks/buffer/mocks.h" #include "test/test_common/network_utility.h" @@ -76,44 +69,6 @@ IntegrationTcpClient::IntegrationTcpClient( connection_->connect(); } -IntegrationTcpClient::IntegrationTcpClient(Event::Dispatcher& dispatcher, - MockBufferFactory& factory, - Network::Address::InstanceConstSharedPtr, - bool enable_half_close, - const Network::ConnectionSocket::OptionsSharedPtr&, - Network::Address::InstanceConstSharedPtr) - : payload_reader_(new WaitForPayloadReader(dispatcher)), - callbacks_(new ConnectionCallbacks(*this)) { - EXPECT_CALL(factory, create_(_, _, _)) - .Times(AtLeast(1)) - .WillOnce(Invoke([&](std::function below_low, std::function above_high, - std::function above_overflow) -> Buffer::Instance* { - client_write_buffer_ = - new NiceMock(below_low, above_high, above_overflow); - return client_write_buffer_; - })) - .WillRepeatedly(Invoke([](std::function below_low, std::function above_high, - std::function above_overflow) -> Buffer::Instance* { - return new Buffer::WatermarkBuffer(below_low, above_high, above_overflow); - })); - auto [io_handle_client, io_handle_server] = - Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); - StreamInfo::StreamInfoImpl stream_info(dispatcher.timeSource(), nullptr); - Network::ConnectionImpl client_connection(dispatcher, - std::make_unique( - std::move(io_handle_client), nullptr, nullptr), - Network::Test::createRawBufferSocket(), stream_info, - false); - - ON_CALL(*client_write_buffer_, drain(_)) - .WillByDefault(Invoke(client_write_buffer_, &MockWatermarkBuffer::trackDrains)); - EXPECT_CALL(*client_write_buffer_, drain(_)).Times(AnyNumber()); - - connection_->enableHalfClose(enable_half_close); - connection_->addConnectionCallbacks(*callbacks_); - connection_->addReadFilter(payload_reader_); -} - void IntegrationTcpClient::close() { connection_->close(Network::ConnectionCloseType::NoFlush); } void IntegrationTcpClient::waitForData(const std::string& data, bool exact_match) { diff --git a/test/integration/integration_tcp_client.h b/test/integration/integration_tcp_client.h index a14c5cd9b4ceb..af75b84de08c1 100644 --- a/test/integration/integration_tcp_client.h +++ b/test/integration/integration_tcp_client.h @@ -30,11 +30,6 @@ class IntegrationTcpClient { Network::Address::IpVersion version, bool enable_half_close, const Network::ConnectionSocket::OptionsSharedPtr& options, Network::Address::InstanceConstSharedPtr source_address = nullptr); - IntegrationTcpClient(Event::Dispatcher& dispatcher, MockBufferFactory& factory, - Network::Address::InstanceConstSharedPtr dest_address, - bool enable_half_close, - const Network::ConnectionSocket::OptionsSharedPtr& options, - Network::Address::InstanceConstSharedPtr source_address = nullptr); void close(); void waitForData(const std::string& data, bool exact_match = true); diff --git a/test/integration/internal_tcp_proxy_integration_test.cc b/test/integration/internal_tcp_proxy_integration_test.cc index fad3a753d9864..7666201d524d3 100644 --- a/test/integration/internal_tcp_proxy_integration_test.cc +++ b/test/integration/internal_tcp_proxy_integration_test.cc @@ -199,7 +199,7 @@ TEST_P(InternalTcpProxyIntegrationTest, DISABLED_TcpProxyUpstreamWritesFirst) { worker_dispatcher_->post([&]() mutable { auto internal_listener = - worker_dispatcher_->getInternalListenerManagerForTest().value().get().findByAddress( + worker_dispatcher_->getInternalListenerManager().value().get().findByAddress( server_address); std::unique_ptr io_handle = std::move(io_handle_server); auto accepted_socket = std::make_unique( diff --git a/test/integration/socket_interface_integration_test.cc b/test/integration/socket_interface_integration_test.cc index bfcf06fc50e7f..4d9449495c721 100644 --- a/test/integration/socket_interface_integration_test.cc +++ b/test/integration/socket_interface_integration_test.cc @@ -108,31 +108,6 @@ TEST_P(SocketInterfaceIntegrationTest, InternalAddressWithSocketInterface) { "panic: not implemented"); } -// Test that connecting to internal address will crash. -TEST_P(SocketInterfaceIntegrationTest, CreateClientConnectionToInternalAddress) { - - BaseIntegrationTest::initialize(); - - ConnectionStatusCallbacks connect_callbacks_; - Network::ClientConnectionPtr client_; - const Network::SocketInterface* sock_interface = Network::socketInterface( - "envoy.extensions.network.socket_interface.default_socket_interface"); - Network::Address::InstanceConstSharedPtr address = - std::make_shared("listener_0", sock_interface); - - Runtime::LoaderSingleton::getExisting()->mergeValues( - {{"envoy.reloadable_features.internal_address", "true"}}); - client_ = dispatcher_->createClientConnection(address, Network::Address::InstanceConstSharedPtr(), - Network::Test::createRawBufferSocket(), nullptr); - client_->addConnectionCallbacks(connect_callbacks_); - client_->connect(); - while (!connect_callbacks_.connected() && !connect_callbacks_.closed()) { - dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - } - ASSERT_TRUE(connect_callbacks_.closed()); - client_->close(Network::ConnectionCloseType::NoFlush); -} - // Test that recv from internal address will crash. // TODO(lambdai): Add UDP internal listener implementation to enable the io path. TEST_P(SocketInterfaceIntegrationTest, UdpRecvFromInternalAddressWithSocketInterface) { diff --git a/test/integration/tcp_proxy_integration_test.h b/test/integration/tcp_proxy_integration_test.h index a458ec960dfa4..9bcaa758b774d 100644 --- a/test/integration/tcp_proxy_integration_test.h +++ b/test/integration/tcp_proxy_integration_test.h @@ -25,6 +25,7 @@ class TcpProxyIntegrationTest : public testing::TestWithParam Date: Tue, 1 Jun 2021 19:58:31 -0700 Subject: [PATCH 03/23] delete test/integration/internal_tcp_proxy_integration_test.cc Signed-off-by: Yuchen Dai --- .../internal_tcp_proxy_integration_test.cc | 903 ------------------ 1 file changed, 903 deletions(-) delete mode 100644 test/integration/internal_tcp_proxy_integration_test.cc diff --git a/test/integration/internal_tcp_proxy_integration_test.cc b/test/integration/internal_tcp_proxy_integration_test.cc deleted file mode 100644 index 7666201d524d3..0000000000000 --- a/test/integration/internal_tcp_proxy_integration_test.cc +++ /dev/null @@ -1,903 +0,0 @@ -#include - -#include "envoy/config/bootstrap/v3/bootstrap.pb.h" -#include "envoy/config/cluster/v3/cluster.pb.h" -#include "envoy/config/core/v3/base.pb.h" -#include "envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.pb.h" -#include "envoy/extensions/access_loggers/file/v3/file.pb.h" -#include "envoy/extensions/filters/network/tcp_proxy/v3/tcp_proxy.pb.h" -#include "envoy/network/address.h" - -#include "common/config/api_version.h" -#include "common/network/address_impl.h" -#include "common/network/utility.h" - -#include "extensions/filters/network/common/factory_base.h" -#include "extensions/io_socket/user_space/io_handle_impl.h" -#include "extensions/transport_sockets/tls/context_manager_impl.h" - -#include "test/integration/ssl_utility.h" -#include "test/integration/tcp_proxy_integration_test.h" -#include "test/integration/tcp_proxy_integration_test.pb.h" -#include "test/integration/tcp_proxy_integration_test.pb.validate.h" -#include "test/integration/utility.h" -#include "test/test_common/registry.h" - -#include "gtest/gtest.h" - -using testing::HasSubstr; - -namespace Envoy { - -class InternalTcpProxyIntegrationTest - : public testing::TestWithParam, - public BaseIntegrationTest { -public: - InternalTcpProxyIntegrationTest() - : BaseIntegrationTest(GetParam().version, internalTcpProxyConfig()) { - enableHalfClose(true); - } - static std::string internalTcpProxyConfig() { - return absl::StrCat(fmt::format(R"EOF( -admin: - access_log: - - name: envoy.access_loggers.file - typed_config: - "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog - path: "{}" - address: - socket_address: - address: 127.0.0.1 - port_value: 0 -dynamic_resources: - lds_config: - resource_api_version: V3 - path: {} -static_resources: - secrets: - - name: "secret_static_0" - tls_certificate: - certificate_chain: - inline_string: "DUMMY_INLINE_BYTES" - private_key: - inline_string: "DUMMY_INLINE_BYTES" - password: - inline_string: "DUMMY_INLINE_BYTES" - clusters: - - name: cluster_0 - load_assignment: - cluster_name: cluster_0 - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 0 - - name: cluster_internal - load_assignment: - cluster_name: cluster_internal - endpoints: - - lb_endpoints: - - endpoint: - address: -# pipe: -# path: "@/tmp/pipetest" - envoy_internal_address: - server_listener_name: test_internal_listener_foo - - listeners: - - name: listener_0 - address: - socket_address: - address: 127.0.0.1 - port_value: 0 - filter_chains: - filters: - name: tcp - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy - stat_prefix: tcp_stats - cluster: cluster_internal - - name: listener_internal - address: -# pipe: -# path: "@/tmp/pipetest" - envoy_internal_address: - server_listener_name: test_internal_listener_foo - internal_listener: {{ }} # escape the bracket -)EOF", - Platform::null_device_path, Platform::null_device_path), - R"EOF( - filter_chains: - filters: - name: tcp - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy - stat_prefix: tcp_stats - cluster: cluster_0 -)EOF"); - } - - void initialize() override; - - Event::Dispatcher* worker_dispatcher_{}; - int chained_number_{2}; -}; - -std::vector newPoolTestParams() { - std::vector ret; - - for (auto ip_version : TestEnvironment::getIpVersionsForTest()) { - ret.push_back(TcpProxyIntegrationTestParams{ip_version, false}); - } - return ret; -} - -std::vector getProtocolTestParams() { - std::vector ret; - - for (auto ip_version : TestEnvironment::getIpVersionsForTest()) { - ret.push_back(TcpProxyIntegrationTestParams{ip_version, true}); - ret.push_back(TcpProxyIntegrationTestParams{ip_version, false}); - } - return ret; -} - -std::string -protocolTestParamsToString(const ::testing::TestParamInfo& params) { - return absl::StrCat( - (params.param.version == Network::Address::IpVersion::v4 ? "IPv4_" : "IPv6_"), - (params.param.test_original_version == true ? "OriginalConnPool" : "NewConnPool")); -} - -void InternalTcpProxyIntegrationTest::initialize() { - if (GetParam().test_original_version) { - config_helper_.addRuntimeOverride("envoy.reloadable_features.new_tcp_connection_pool", "false"); - } else { - config_helper_.addRuntimeOverride("envoy.reloadable_features.new_tcp_connection_pool", "true"); - } - - config_helper_.renameListener("tcp_proxy"); - BaseIntegrationTest::initialize(); - - // worker_dispatcher_ can only be obtained after initialization. - worker_dispatcher_ = getWorkerDispatcher(1); - ASSERT(worker_dispatcher_ != nullptr); -} - -// The internal connection clients are driven by the dispatcher. -INSTANTIATE_TEST_SUITE_P(TcpProxyIntegrationTestParams, InternalTcpProxyIntegrationTest, - testing::ValuesIn(getProtocolTestParams()), protocolTestParamsToString); - -// There are two tcp proxies in this test suite. -// -// The first tcp proxy is owned by the tcp listener. This tcp proxy handles the data from test main -// thread and create upstream internal connection to the internal listener. -// -// The second tcp proxy is owned by the internal listener. This tcp proxy relay the data to the fake -// upstream. -using ChainedProxyInternalTcpProxyIntegrationTest = InternalTcpProxyIntegrationTest; -INSTANTIATE_TEST_SUITE_P(TcpProxyIntegrationTestParams, ChainedProxyInternalTcpProxyIntegrationTest, - testing::ValuesIn(getProtocolTestParams()), protocolTestParamsToString); - -// TODO(lambdai): enable it when the io_handle_client can be gracefully destroyed in the worker -// thread. -TEST_P(InternalTcpProxyIntegrationTest, DISABLED_TcpProxyUpstreamWritesFirst) { - initialize(); - - // lambda-expression cannot captures a structured binding so we must use tie() here. - Network::IoHandlePtr io_handle_client; - Network::IoHandlePtr io_handle_server; - - std::tie(io_handle_client, io_handle_server) = - Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); - - auto server_address = - std::make_shared("test_internal_listener_foo"); - auto client_address = std::make_shared("client_bar"); - - worker_dispatcher_->post([&]() mutable { - auto internal_listener = - worker_dispatcher_->getInternalListenerManager().value().get().findByAddress( - server_address); - std::unique_ptr io_handle = std::move(io_handle_server); - auto accepted_socket = std::make_unique( - std::move(io_handle), server_address, client_address); - internal_listener.value().get().onAccept(std::move(accepted_socket)); - }); - - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - - ASSERT_TRUE(fake_upstream_connection->write("hello")); - - Buffer::OwnedImpl buffer; - Thread::MutexBasicLockable dispatcher_mutex; - - while (buffer.length() < 5) { - Thread::CondVar post_complete; - worker_dispatcher_->post([&]() { - Thread::LockGuard guard(dispatcher_mutex); - io_handle_client->read(buffer, absl::nullopt); - post_complete.notifyOne(); - }); - - Thread::LockGuard guard(dispatcher_mutex); - post_complete.wait(dispatcher_mutex); - ENVOY_LOG_MISC(debug, "current buffer: {}", buffer.toString()); - } - ASSERT_TRUE(fake_upstream_connection->write("", true)); -} - -// Test upstream writing before downstream downstream does. -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyUpstreamWritesFirst) { - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - - ASSERT_TRUE(fake_upstream_connection->write("hello")); - tcp_client->waitForData("hello"); - // Make sure inexact matches work also on data already received. - tcp_client->waitForData("ello", false); - - // Make sure length based wait works for the data already received - ASSERT_TRUE(tcp_client->waitForData(5)); - ASSERT_TRUE(tcp_client->waitForData(4)); - - // Drain part of the received message - tcp_client->clearData(2); - tcp_client->waitForData("llo"); - ASSERT_TRUE(tcp_client->waitForData(3)); - - ASSERT_TRUE(tcp_client->write("hello")); - ASSERT_TRUE(fake_upstream_connection->waitForData(5)); - - ASSERT_TRUE(fake_upstream_connection->write("", true)); - tcp_client->waitForHalfClose(); - ASSERT_TRUE(tcp_client->write("", true)); - ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - // Any time an associated connection is destroyed, it increments both counters. - test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_destroy", 1); - test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_destroy_with_active_rq", 1); - - IntegrationTcpClientPtr tcp_client2 = makeTcpConnection(lookupPort("tcp_proxy")); - FakeRawConnectionPtr fake_upstream_connection2; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection2)); - tcp_client2->close(); -} - -// Test TLS upstream. -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyUpstreamTls) { - upstream_tls_ = true; - setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); - config_helper_.configureUpstreamTls(); - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - ASSERT_TRUE(tcp_client->write("hello")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - ASSERT_TRUE(fake_upstream_connection->waitForData(5)); - ASSERT_TRUE(fake_upstream_connection->write("world")); - ASSERT_TRUE(fake_upstream_connection->close()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - tcp_client->waitForHalfClose(); - tcp_client->close(); - - EXPECT_EQ("world", tcp_client->data()); - // Any time an associated connection is destroyed, it increments both counters. - test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_destroy", 1); - test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_destroy_with_active_rq", 1); -} - -// Test proxying data in both directions, and that all data is flushed properly -// when there is an upstream disconnect. -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyUpstreamDisconnect) { - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - ASSERT_TRUE(tcp_client->write("hello")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - ASSERT_TRUE(fake_upstream_connection->waitForData(5)); - ASSERT_TRUE(fake_upstream_connection->write("world")); - ASSERT_TRUE(fake_upstream_connection->close()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - tcp_client->waitForHalfClose(); - tcp_client->close(); - - EXPECT_EQ("world", tcp_client->data()); -} - -// Test proxying data in both directions, and that all data is flushed properly -// when the client disconnects. -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyDownstreamDisconnect) { - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - ASSERT_TRUE(tcp_client->write("hello")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - ASSERT_TRUE(fake_upstream_connection->waitForData(5)); - ASSERT_TRUE(fake_upstream_connection->write("world")); - tcp_client->waitForData("world"); - ASSERT_TRUE(tcp_client->write("hello", true)); - ASSERT_TRUE(fake_upstream_connection->waitForData(10)); - ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); - ASSERT_TRUE(fake_upstream_connection->write("", true)); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - tcp_client->waitForDisconnect(); -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyManyConnections) { - autonomous_upstream_ = true; - initialize(); - const int num_connections = 50; - std::vector clients(num_connections); - - for (int i = 0; i < num_connections; ++i) { - clients[i] = makeTcpConnection(lookupPort("tcp_proxy")); - } - test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_total", num_connections); - for (int i = 0; i < num_connections; ++i) { - IntegrationTcpClientPtr& tcp_client = clients[i]; - // The autonomous upstream is an HTTP upstream, so send raw HTTP. - // This particular request will result in the upstream sending a response, - // and flush-closing due to the 'close_after_response' header. - ASSERT_TRUE(tcp_client->write( - "GET / HTTP/1.1\r\nHost: foo\r\nclose_after_response: yes\r\ncontent-length: 0\r\n\r\n", - false)); - tcp_client->waitForHalfClose(); - tcp_client->close(); - EXPECT_THAT(tcp_client->data(), HasSubstr("aaaaaaaaaa")); - } -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyRandomBehavior) { - autonomous_upstream_ = true; - initialize(); - std::list clients; - - // The autonomous upstream parses HTTP, and HTTP headers and sends responses - // when full requests are received. basic_request will result in - // bidirectional data. request_with_close will result in bidirectional data, - // but also the upstream closing the connection. - const char* basic_request = "GET / HTTP/1.1\r\nHost: foo\r\ncontent-length: 0\r\n\r\n"; - const char* request_with_close = - "GET / HTTP/1.1\r\nHost: foo\r\nclose_after_response: yes\r\ncontent-length: 0\r\n\r\n"; - TestRandomGenerator rand; - - // Seed some initial clients - for (int i = 0; i < 5; ++i) { - clients.push_back(makeTcpConnection(lookupPort("tcp_proxy"))); - } - - // Now randomly write / add more connections / close. - for (int i = 0; i < 50; ++i) { - int action = rand.random() % 3; - - if (action == 0) { - // Add a new connection. - clients.push_back(makeTcpConnection(lookupPort("tcp_proxy"))); - } - if (clients.empty()) { - break; - } - IntegrationTcpClientPtr& tcp_client = clients.front(); - if (action == 1) { - // Write to the first connection. - ASSERT_TRUE(tcp_client->write(basic_request, false)); - tcp_client->waitForData("\r\n\r\n", false); - tcp_client->clearData(tcp_client->data().size()); - } else if (action == 2) { - // Close the first connection. - ASSERT_TRUE(tcp_client->write(request_with_close, false)); - tcp_client->waitForData("\r\n\r\n", false); - tcp_client->waitForHalfClose(); - tcp_client->close(); - clients.pop_front(); - } - } - - while (!clients.empty()) { - IntegrationTcpClientPtr& tcp_client = clients.front(); - ASSERT_TRUE(tcp_client->write(request_with_close, false)); - tcp_client->waitForData("\r\n\r\n", false); - tcp_client->waitForHalfClose(); - tcp_client->close(); - clients.pop_front(); - } -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, NoUpstream) { - // Set the first upstream to have an invalid port, so connection will fail, - // but it won't fail synchronously (as it would if there were simply no - // upstreams) - fake_upstreams_count_ = 0; - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - auto* cluster = bootstrap.mutable_static_resources()->mutable_clusters(0); - auto* lb_endpoint = - cluster->mutable_load_assignment()->mutable_endpoints(0)->mutable_lb_endpoints(0); - lb_endpoint->mutable_endpoint()->mutable_address()->mutable_socket_address()->set_port_value(1); - }); - config_helper_.skipPortUsageValidation(); - enableHalfClose(false); - initialize(); - - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - tcp_client->waitForDisconnect(); -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyLargeWrite) { - config_helper_.setBufferLimits(1024, 1024); - initialize(); - - std::string data(1024 * 16, 'a'); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - ASSERT_TRUE(tcp_client->write(data)); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - ASSERT_TRUE(fake_upstream_connection->waitForData(data.size())); - ASSERT_TRUE(fake_upstream_connection->write(data)); - tcp_client->waitForData(data); - tcp_client->close(); - ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); - ASSERT_TRUE(fake_upstream_connection->close()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - - uint32_t upstream_pauses = - test_server_->counter("cluster.cluster_0.upstream_flow_control_paused_reading_total") - ->value(); - uint32_t upstream_resumes = - test_server_->counter("cluster.cluster_0.upstream_flow_control_resumed_reading_total") - ->value(); - EXPECT_EQ(upstream_pauses, upstream_resumes); - - uint32_t downstream_pauses = - test_server_->counter("tcp.tcp_stats.downstream_flow_control_paused_reading_total")->value(); - uint32_t downstream_resumes = - test_server_->counter("tcp.tcp_stats.downstream_flow_control_resumed_reading_total")->value(); - EXPECT_EQ(downstream_pauses, downstream_resumes); -} - -// Test that a downstream flush works correctly (all data is flushed) -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyDownstreamFlush) { - // Use a very large size to make sure it is larger than the kernel socket read buffer. - const uint32_t size = 50 * 1024 * 1024; - config_helper_.setBufferLimits(size / 8, size / 8); - initialize(); - - std::string data(size, 'a'); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - tcp_client->readDisable(true); - ASSERT_TRUE(tcp_client->write("", true)); - - // This ensures that readDisable(true) has been run on it's thread - // before tcp_client starts writing. - ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); - - ASSERT_TRUE(fake_upstream_connection->write(data, true)); - - // test_server_->waitForCounterGe("cluster.cluster_0.upstream_flow_control_paused_reading_total", - // 1); - // EXPECT_EQ(test_server_->counter("cluster.cluster_0.upstream_flow_control_resumed_reading_total") - // ->value(), - // 0); - tcp_client->readDisable(false); - tcp_client->waitForData(data); - tcp_client->waitForHalfClose(); - ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); - - uint32_t upstream_pauses = - test_server_->counter("cluster.cluster_0.upstream_flow_control_paused_reading_total") - ->value(); - uint32_t upstream_resumes = - test_server_->counter("cluster.cluster_0.upstream_flow_control_resumed_reading_total") - ->value(); - EXPECT_GE(upstream_pauses, upstream_resumes); - EXPECT_GT(upstream_resumes, 0); -} - -// Test that an upstream flush works correctly (all data is flushed) -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyUpstreamFlush) { - // Use a very large size to make sure it is larger than the kernel socket read buffer. - const uint32_t size = 1 * 1024 * 1024; - // config_helper_.setBufferLimits(size / 2, size / 2); - initialize(); - - std::string data(size, 'a'); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - ASSERT_TRUE(fake_upstream_connection->readDisable(true)); - ASSERT_TRUE(fake_upstream_connection->write("", true)); - - // This ensures that fake_upstream_connection->readDisable has been run on it's thread - // before tcp_client starts writing. - tcp_client->waitForHalfClose(); - - ASSERT_TRUE(tcp_client->write(data, true, true, std::chrono::milliseconds(30000))); - FANCY_LOG(debug, "lambdai: waiting for upstream_flush_active"); - // test_server_->waitForGaugeGe("tcp.tcp_stats.upstream_flush_active", 1); - ASSERT_TRUE(fake_upstream_connection->readDisable(false)); - ASSERT_TRUE(fake_upstream_connection->waitForData(data.size())); - ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - // tcp_client->waitForDisconnect(true); - FANCY_LOG(debug, "client disconnected"); - // Double because two tcp proxy are chained. - EXPECT_EQ(test_server_->counter("tcp.tcp_stats.upstream_flush_total")->value(), chained_number_); - FANCY_LOG(debug, "flush total == {} ", - test_server_->counter("tcp.tcp_stats.upstream_flush_total")->value()); - test_server_->waitForGaugeEq("tcp.tcp_stats.upstream_flush_active", 0); -} - -// Test that Envoy doesn't crash or assert when shutting down with an upstream flush active -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TcpProxyUpstreamFlushEnvoyExit) { - // Use a very large size to make sure it is larger than the kernel socket read buffer. - const uint32_t size = 50 * 1024 * 1024; - config_helper_.setBufferLimits(size, size); - initialize(); - - std::string data(size, 'a'); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - ASSERT_TRUE(fake_upstream_connection->readDisable(true)); - ASSERT_TRUE(fake_upstream_connection->write("", true)); - - // This ensures that fake_upstream_connection->readDisable has been run on it's thread - // before tcp_client starts writing. - tcp_client->waitForHalfClose(); - - ASSERT_TRUE(tcp_client->write(data, true)); - - test_server_->waitForGaugeEq("tcp.tcp_stats.upstream_flush_active", 1); - test_server_.reset(); - ASSERT_TRUE(fake_upstream_connection->close()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - - // Success criteria is that no ASSERTs fire and there are no leaks. -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, AccessLog) { - std::string access_log_path = TestEnvironment::temporaryPath( - fmt::format("access_log{}.txt", version_ == Network::Address::IpVersion::v4 ? "v4" : "v6")); - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); - auto* filter_chain = listener->mutable_filter_chains(0); - auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); - - ASSERT_TRUE(config_blob->Is()); - auto tcp_proxy_config = - MessageUtil::anyConvert( - *config_blob); - - auto* access_log = tcp_proxy_config.add_access_log(); - access_log->set_name("accesslog"); - envoy::extensions::access_loggers::file::v3::FileAccessLog access_log_config; - access_log_config.set_path(access_log_path); - access_log_config.mutable_log_format()->mutable_text_format_source()->set_inline_string( - "upstreamlocal=%UPSTREAM_LOCAL_ADDRESS% " - "upstreamhost=%UPSTREAM_HOST% downstream=%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT% " - "sent=%BYTES_SENT% received=%BYTES_RECEIVED%\n"); - access_log->mutable_typed_config()->PackFrom(access_log_config); - auto* runtime_filter = access_log->mutable_filter()->mutable_runtime_filter(); - runtime_filter->set_runtime_key("unused-key"); - auto* percent_sampled = runtime_filter->mutable_percent_sampled(); - percent_sampled->set_numerator(100); - percent_sampled->set_denominator(envoy::type::v3::FractionalPercent::DenominatorType:: - FractionalPercent_DenominatorType_HUNDRED); - config_blob->PackFrom(tcp_proxy_config); - }); - initialize(); - - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - - ASSERT_TRUE(fake_upstream_connection->write("hello")); - tcp_client->waitForData("hello"); - - ASSERT_TRUE(fake_upstream_connection->write("", true)); - tcp_client->waitForHalfClose(); - ASSERT_TRUE(tcp_client->write("", true)); - ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - - std::string log_result; - // Access logs only get flushed to disk periodically, so poll until the log is non-empty - do { - log_result = api_->fileSystem().fileReadToEnd(access_log_path); - } while (log_result.empty()); - - // Regex matching localhost:port -#ifndef GTEST_USES_SIMPLE_RE - const std::string ip_port_regex = (version_ == Network::Address::IpVersion::v4) - ? R"EOF(127\.0\.0\.1:[0-9]+)EOF" - : R"EOF(\[::1\]:[0-9]+)EOF"; -#else - const std::string ip_port_regex = (version_ == Network::Address::IpVersion::v4) - ? R"EOF(127\.0\.0\.1:\d+)EOF" - : R"EOF(\[::1\]:\d+)EOF"; -#endif - - const std::string ip_regex = - (version_ == Network::Address::IpVersion::v4) ? R"EOF(127.0.0.1)EOF" : R"EOF(::1)EOF"; - - // Test that all three addresses were populated correctly. Only check the first line of - // log output for simplicity. - EXPECT_THAT(log_result, HasSubstr(fmt::format( - "upstreamlocal={0} upstreamhost={2} downstream={1} sent=5 received=0", - // Upstream local is not defined for internal connection. - "-", ip_regex, - // Upstream remote address is formatted as internal listener name. - "envoy://test_internal_listener_foo"))); -} - -// Test that the server shuts down without crashing when connections are open. -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, ShutdownWithOpenConnections) { - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - auto* static_resources = bootstrap.mutable_static_resources(); - for (int i = 0; i < static_resources->clusters_size(); ++i) { - auto* cluster = static_resources->mutable_clusters(i); - cluster->set_close_connections_on_host_health_failure(true); - } - }); - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - ASSERT_TRUE(tcp_client->write("hello")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - ASSERT_TRUE(fake_upstream_connection->waitForData(5)); - ASSERT_TRUE(fake_upstream_connection->write("world")); - tcp_client->waitForData("world"); - ASSERT_TRUE(tcp_client->write("hello", false)); - ASSERT_TRUE(fake_upstream_connection->waitForData(10)); - test_server_.reset(); - ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); - ASSERT_TRUE(fake_upstream_connection->close()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - tcp_client->waitForHalfClose(); - tcp_client->close(); - - // Success criteria is that no ASSERTs fire and there are no leaks. -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TestIdletimeoutWithNoData) { - autonomous_upstream_ = true; - - enableHalfClose(false); - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); - auto* filter_chain = listener->mutable_filter_chains(0); - auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); - - ASSERT_TRUE(config_blob->Is()); - auto tcp_proxy_config = - MessageUtil::anyConvert( - *config_blob); - tcp_proxy_config.mutable_idle_timeout()->set_nanos( - std::chrono::duration_cast(std::chrono::milliseconds(100)) - .count()); - config_blob->PackFrom(tcp_proxy_config); - }); - - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - tcp_client->waitForDisconnect(); -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TestIdletimeoutWithLargeOutstandingData) { - config_helper_.setBufferLimits(1024, 1024); - enableHalfClose(false); - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); - auto* filter_chain = listener->mutable_filter_chains(0); - auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); - - ASSERT_TRUE(config_blob->Is()); - auto tcp_proxy_config = - MessageUtil::anyConvert( - *config_blob); - tcp_proxy_config.mutable_idle_timeout()->set_nanos( - std::chrono::duration_cast(std::chrono::milliseconds(500)) - .count()); - config_blob->PackFrom(tcp_proxy_config); - }); - - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - - std::string data(1024 * 16, 'a'); - ASSERT_TRUE(tcp_client->write(data)); - ASSERT_TRUE(fake_upstream_connection->write(data)); - - tcp_client->waitForDisconnect(); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, TestMaxDownstreamConnectionDurationWithNoData) { - autonomous_upstream_ = true; - - enableHalfClose(false); - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); - auto* filter_chain = listener->mutable_filter_chains(0); - auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); - - ASSERT_TRUE(config_blob->Is()); - auto tcp_proxy_config = - MessageUtil::anyConvert( - *config_blob); - tcp_proxy_config.mutable_max_downstream_connection_duration()->set_nanos( - std::chrono::duration_cast(std::chrono::milliseconds(100)) - .count()); - config_blob->PackFrom(tcp_proxy_config); - }); - - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - tcp_client->waitForDisconnect(); -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, - TestMaxDownstreamConnectionDurationWithLargeOutstandingData) { - config_helper_.setBufferLimits(1024, 1024); - enableHalfClose(false); - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); - auto* filter_chain = listener->mutable_filter_chains(0); - auto* config_blob = filter_chain->mutable_filters(0)->mutable_typed_config(); - - ASSERT_TRUE(config_blob->Is()); - auto tcp_proxy_config = - MessageUtil::anyConvert( - *config_blob); - tcp_proxy_config.mutable_max_downstream_connection_duration()->set_nanos( - std::chrono::duration_cast(std::chrono::milliseconds(500)) - .count()); - config_blob->PackFrom(tcp_proxy_config); - }); - - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - - std::string data(1024 * 16, 'a'); - ASSERT_TRUE(tcp_client->write(data)); - ASSERT_TRUE(fake_upstream_connection->write(data)); - - tcp_client->waitForDisconnect(); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, DISABLED_TestNoCloseOnHealthFailure) { - concurrency_ = 2; - - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - auto* static_resources = bootstrap.mutable_static_resources(); - for (int i = 0; i < static_resources->clusters_size(); ++i) { - auto* cluster = static_resources->mutable_clusters(i); - cluster->set_close_connections_on_host_health_failure(false); - cluster->mutable_common_lb_config()->mutable_healthy_panic_threshold()->set_value(0); - cluster->add_health_checks()->mutable_timeout()->set_seconds(20); - cluster->mutable_health_checks(0)->mutable_reuse_connection()->set_value(true); - cluster->mutable_health_checks(0)->mutable_interval()->set_seconds(1); - cluster->mutable_health_checks(0)->mutable_no_traffic_interval()->set_seconds(1); - cluster->mutable_health_checks(0)->mutable_unhealthy_threshold()->set_value(1); - cluster->mutable_health_checks(0)->mutable_healthy_threshold()->set_value(1); - cluster->mutable_health_checks(0)->mutable_tcp_health_check(); - cluster->mutable_health_checks(0)->mutable_tcp_health_check()->mutable_send()->set_text( - "50696E67"); - cluster->mutable_health_checks(0)->mutable_tcp_health_check()->add_receive()->set_text( - "506F6E67"); - } - }); - - FakeRawConnectionPtr fake_upstream_health_connection; - on_server_init_function_ = [&](void) -> void { - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_health_connection)); - ASSERT_TRUE(fake_upstream_health_connection->waitForData( - FakeRawConnection::waitForInexactMatch("Ping"))); - ASSERT_TRUE(fake_upstream_health_connection->write("Pong")); - }; - - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - ASSERT_TRUE(tcp_client->write("hello")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - ASSERT_TRUE(fake_upstream_connection->waitForData(5)); - ASSERT_TRUE(fake_upstream_connection->write("world")); - tcp_client->waitForData("world"); - ASSERT_TRUE(tcp_client->write("hello")); - ASSERT_TRUE(fake_upstream_connection->waitForData(10)); - - ASSERT_TRUE(fake_upstream_health_connection->waitForData(8)); - ASSERT_TRUE(fake_upstream_health_connection->close()); - ASSERT_TRUE(fake_upstream_health_connection->waitForDisconnect()); - - // By waiting we know the previous health check attempt completed (with a failure since we closed - // the connection on it) - FakeRawConnectionPtr fake_upstream_health_connection_reconnect; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_health_connection_reconnect)); - ASSERT_TRUE(fake_upstream_health_connection_reconnect->waitForData( - FakeRawConnection::waitForInexactMatch("Ping"))); - - ASSERT_TRUE(tcp_client->write("still")); - ASSERT_TRUE(fake_upstream_connection->waitForData(15)); - ASSERT_TRUE(fake_upstream_connection->write("here")); - tcp_client->waitForData("here", false); - - test_server_.reset(); - ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); - ASSERT_TRUE(fake_upstream_connection->close()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - ASSERT_TRUE(fake_upstream_health_connection_reconnect->waitForHalfClose()); - ASSERT_TRUE(fake_upstream_health_connection_reconnect->close()); - ASSERT_TRUE(fake_upstream_health_connection_reconnect->waitForDisconnect()); - tcp_client->waitForHalfClose(); - tcp_client->close(); -} - -TEST_P(ChainedProxyInternalTcpProxyIntegrationTest, DISABLED_TestCloseOnHealthFailure) { - concurrency_ = 2; - - config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { - auto* static_resources = bootstrap.mutable_static_resources(); - for (int i = 0; i < static_resources->clusters_size(); ++i) { - auto* cluster = static_resources->mutable_clusters(i); - cluster->set_close_connections_on_host_health_failure(true); - cluster->mutable_common_lb_config()->mutable_healthy_panic_threshold()->set_value(0); - cluster->add_health_checks()->mutable_timeout()->set_seconds(20); - cluster->mutable_health_checks(0)->mutable_reuse_connection()->set_value(true); - cluster->mutable_health_checks(0)->mutable_interval()->set_seconds(1); - cluster->mutable_health_checks(0)->mutable_no_traffic_interval()->set_seconds(1); - cluster->mutable_health_checks(0)->mutable_unhealthy_threshold()->set_value(1); - cluster->mutable_health_checks(0)->mutable_healthy_threshold()->set_value(1); - cluster->mutable_health_checks(0)->mutable_tcp_health_check(); - cluster->mutable_health_checks(0)->mutable_tcp_health_check()->mutable_send()->set_text( - "50696E67"); - ; - cluster->mutable_health_checks(0)->mutable_tcp_health_check()->add_receive()->set_text( - "506F6E67"); - } - }); - - FakeRawConnectionPtr fake_upstream_health_connection; - on_server_init_function_ = [&](void) -> void { - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_health_connection)); - ASSERT_TRUE(fake_upstream_health_connection->waitForData(4)); - ASSERT_TRUE(fake_upstream_health_connection->write("Pong")); - }; - - initialize(); - IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - ASSERT_TRUE(tcp_client->write("hello")); - FakeRawConnectionPtr fake_upstream_connection; - ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); - ASSERT_TRUE(fake_upstream_connection->waitForData(5)); - ASSERT_TRUE(fake_upstream_connection->write("world")); - tcp_client->waitForData("world"); - ASSERT_TRUE(tcp_client->write("hello")); - ASSERT_TRUE(fake_upstream_connection->waitForData(10)); - - ASSERT_TRUE(fake_upstream_health_connection->waitForData(8)); - ASSERT_TRUE(fake_upstream_health_connection->close()); - ASSERT_TRUE(fake_upstream_health_connection->waitForDisconnect()); - - ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); - tcp_client->waitForHalfClose(); - - ASSERT_TRUE(fake_upstream_connection->close()); - tcp_client->close(); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); -} - -} // namespace Envoy From 635f185266ab3b16be7fa404279d21aae0a4564e Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 1 Jun 2021 23:11:43 -0700 Subject: [PATCH 04/23] coverage Signed-off-by: Yuchen Dai --- test/common/network/connection_impl_test.cc | 2 ++ .../io_socket/user_space/io_handle_impl_test.cc | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 832cae04eb75f..df3f37492aec0 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -2981,6 +2981,8 @@ TEST_F(InternalClientConnectionImplTest, MockInternalListenerManager internal_listener_manager; dispatcher_->registerInternalListenerManager(internal_listener_manager); + const auto& test_internal_listener_manager = dispatcher_->getInternalListenerManager(); + ASSERT_EQ(&internal_listener_manager, &test_internal_listener_manager.value().get()); // Not implemented yet. ASSERT_DEATH( { diff --git a/test/extensions/io_socket/user_space/io_handle_impl_test.cc b/test/extensions/io_socket/user_space/io_handle_impl_test.cc index 790c7a0734ed4..76cc5cb0a0811 100644 --- a/test/extensions/io_socket/user_space/io_handle_impl_test.cc +++ b/test/extensions/io_socket/user_space/io_handle_impl_test.cc @@ -1010,6 +1010,15 @@ TEST_F(IoHandleImplTest, Connect) { EXPECT_EQ(0, io_handle_->connect(address_is_ignored).rc_); } +TEST_F(IoHandleImplTest, ConnectToClosedIoHandle) { + auto address_is_ignored = + std::make_shared("listener_id"); + io_handle_peer_->close(); + auto result = io_handle_->connect(address_is_ignored); + EXPECT_EQ(-1, result.rc_); + EXPECT_EQ(SOCKET_ERROR_INVAL, result.errno_); +} + TEST_F(IoHandleImplTest, ActivateEvent) { schedulable_cb_ = new NiceMock(&dispatcher_); io_handle_->initializeFileEvent( From 3615512a9df16ad3b854e96b9a7fee9576f25e1c Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Jun 2021 15:28:44 -0700 Subject: [PATCH 05/23] remove ClientConnectionImpl ctor for internal socket Signed-off-by: Yuchen Dai --- source/common/network/connection_impl.cc | 15 --------------- source/common/network/connection_impl.h | 8 -------- 2 files changed, 23 deletions(-) diff --git a/source/common/network/connection_impl.cc b/source/common/network/connection_impl.cc index 59f20a109b079..03a3311ae2123 100644 --- a/source/common/network/connection_impl.cc +++ b/source/common/network/connection_impl.cc @@ -819,21 +819,6 @@ void ServerConnectionImpl::onTransportSocketConnectTimeout() { closeConnectionImmediately(); } -ClientConnectionImpl::ClientConnectionImpl( - Event::Dispatcher& dispatcher, std::unique_ptr client_io_handle, - const Network::Address::InstanceConstSharedPtr& address, - const Network::Address::InstanceConstSharedPtr& source_address, - Network::TransportSocketPtr&& transport_socket, - const Network::ConnectionSocket::OptionsSharedPtr& options) - : ConnectionImpl( - dispatcher, - std::make_unique(std::move(client_io_handle), nullptr, address), - std::move(transport_socket), stream_info_, false), - stream_info_(dispatcher.timeSource(), socket_->addressProviderSharedPtr()) { - UNREFERENCED_PARAMETER(source_address); - UNREFERENCED_PARAMETER(options); -} - ClientConnectionImpl::ClientConnectionImpl( Event::Dispatcher& dispatcher, const Address::InstanceConstSharedPtr& remote_address, const Network::Address::InstanceConstSharedPtr& source_address, diff --git a/source/common/network/connection_impl.h b/source/common/network/connection_impl.h index 1c08493226054..8586b2f5707e8 100644 --- a/source/common/network/connection_impl.h +++ b/source/common/network/connection_impl.h @@ -239,14 +239,6 @@ class ClientConnectionImpl : public ConnectionImpl, virtual public ClientConnect const Address::InstanceConstSharedPtr& source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options); - // Internal socket is manufactured by its own factory. - ClientConnectionImpl(Event::Dispatcher& dispatcher, - std::unique_ptr client_io_handle, - const Network::Address::InstanceConstSharedPtr& address, - const Network::Address::InstanceConstSharedPtr& source_address, - Network::TransportSocketPtr&& transport_socket, - const Network::ConnectionSocket::OptionsSharedPtr& options); - // Network::ClientConnection void connect() override; From efe9ce7007780bffda95fe2a085641a78009b785 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 9 Jun 2021 21:59:50 +0000 Subject: [PATCH 06/23] revert dispatcher Signed-off-by: Yuchen Dai --- include/envoy/event/dispatcher.h | 7 ------- source/common/event/dispatcher_impl.cc | 10 ---------- source/common/event/dispatcher_impl.h | 6 ------ source/server/worker_impl.cc | 1 - test/common/network/connection_impl_test.cc | 5 ----- test/mocks/event/mocks.h | 10 ---------- test/mocks/event/wrapped_dispatcher.h | 9 --------- 7 files changed, 48 deletions(-) diff --git a/include/envoy/event/dispatcher.h b/include/envoy/event/dispatcher.h index 24b9f1a8bd502..1618c9236d178 100644 --- a/include/envoy/event/dispatcher.h +++ b/include/envoy/event/dispatcher.h @@ -202,13 +202,6 @@ class Dispatcher : public DispatcherBase { Network::Address::InstanceConstSharedPtr source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) PURE; - /** - * Register an internal listener manager for this dispatcher. - */ - virtual void - registerInternalListenerManager(Network::InternalListenerManager& internal_listener_manager) PURE; - - virtual Network::InternalListenerManagerOptRef getInternalListenerManager() PURE; /** * Creates an async DNS resolver. The resolver should only be used on the thread that runs this diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 60e624d2cf2cd..b3ff5128eb781 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -214,16 +214,6 @@ DispatcherImpl::createUdpListener(Network::SocketSharedPtr socket, config); } -void DispatcherImpl::registerInternalListenerManager( - Network::InternalListenerManager& internal_listener_manager) { - ASSERT(!internal_listener_manager_.has_value()); - internal_listener_manager_ = internal_listener_manager; -} - -Network::InternalListenerManagerOptRef DispatcherImpl::getInternalListenerManager() { - return internal_listener_manager_; -} - TimerPtr DispatcherImpl::createTimer(TimerCb cb) { ASSERT(isThreadSafe()); return createTimerInternal(cb); diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index 4b2965da4bcd4..fe16145e7cd2d 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -78,11 +78,6 @@ class DispatcherImpl : Logger::Loggable, Network::UdpListenerPtr createUdpListener(Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb, const envoy::config::core::v3::UdpSocketConfig& config) override; - - void registerInternalListenerManager( - Network::InternalListenerManager& internal_listener_manager) override; - Network::InternalListenerManagerOptRef getInternalListenerManager() override; - TimerPtr createTimer(TimerCb cb) override; TimerPtr createScaledTimer(ScaledTimerType timer_type, TimerCb cb) override; TimerPtr createScaledTimer(ScaledTimerMinimum minimum, TimerCb cb) override; @@ -180,7 +175,6 @@ class DispatcherImpl : Logger::Loggable, MonotonicTime approximate_monotonic_time_; WatchdogRegistrationPtr watchdog_registration_; const ScaledRangeTimerManagerPtr scaled_timer_manager_; - Network::InternalListenerManagerOptRef internal_listener_manager_; }; } // namespace Event diff --git a/source/server/worker_impl.cc b/source/server/worker_impl.cc index 043a4e7a9cc1e..68fd72defdb99 100644 --- a/source/server/worker_impl.cc +++ b/source/server/worker_impl.cc @@ -19,7 +19,6 @@ WorkerPtr ProdWorkerFactory::createWorker(uint32_t index, OverloadManager& overl Event::DispatcherPtr dispatcher( api_.allocateDispatcher(worker_name, overload_manager.scaledTimerFactory())); auto conn_handler = std::make_unique(*dispatcher, index); - dispatcher->registerInternalListenerManager(*conn_handler); return std::make_unique(tls_, hooks_, std::move(dispatcher), std::move(conn_handler), overload_manager, api_); } diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index df3f37492aec0..8299283668bc3 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -2978,11 +2978,6 @@ TEST_F(InternalClientConnectionImplTest, "envoy.extensions.network.socket_interface.default_socket_interface"); Network::Address::InstanceConstSharedPtr address = std::make_shared("listener_0", sock_interface); - MockInternalListenerManager internal_listener_manager; - - dispatcher_->registerInternalListenerManager(internal_listener_manager); - const auto& test_internal_listener_manager = dispatcher_->getInternalListenerManager(); - ASSERT_EQ(&internal_listener_manager, &test_internal_listener_manager.value().get()); // Not implemented yet. ASSERT_DEATH( { diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index 5dcfc22922bce..36544bd2c1d2f 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -77,14 +77,6 @@ class MockDispatcher : public Dispatcher { const envoy::config::core::v3::UdpSocketConfig& config) override { return Network::UdpListenerPtr{createUdpListener_(socket, cb, config)}; } - void registerInternalListenerManager( - Network::InternalListenerManager& internal_listener_manager) override { - return registerInternalListenerManager_(internal_listener_manager); - } - - Network::InternalListenerManagerOptRef getInternalListenerManager() override { - return getInternalListenerManager_(); - } Event::TimerPtr createTimer(Event::TimerCb cb) override { auto timer = Event::TimerPtr{createTimer_(cb)}; @@ -150,8 +142,6 @@ class MockDispatcher : public Dispatcher { MOCK_METHOD(Network::UdpListener*, createUdpListener_, (Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb, const envoy::config::core::v3::UdpSocketConfig& config)); - MOCK_METHOD(void, registerInternalListenerManager_, (Network::InternalListenerManager&)); - MOCK_METHOD(Network::InternalListenerManagerOptRef, getInternalListenerManager_, ()); MOCK_METHOD(Timer*, createTimer_, (Event::TimerCb cb)); MOCK_METHOD(Timer*, createScaledTimer_, (ScaledTimerMinimum minimum, Event::TimerCb cb)); MOCK_METHOD(Timer*, createScaledTypedTimer_, (ScaledTimerType timer_type, Event::TimerCb cb)); diff --git a/test/mocks/event/wrapped_dispatcher.h b/test/mocks/event/wrapped_dispatcher.h index 6c97ef09e90dd..776f0a5035ec5 100644 --- a/test/mocks/event/wrapped_dispatcher.h +++ b/test/mocks/event/wrapped_dispatcher.h @@ -77,15 +77,6 @@ class WrappedDispatcher : public Dispatcher { return impl_.createUdpListener(std::move(socket), cb, config); } - void registerInternalListenerManager( - Network::InternalListenerManager& internal_listener_manager) override { - impl_.registerInternalListenerManager(internal_listener_manager); - } - - Network::InternalListenerManagerOptRef getInternalListenerManager() override { - return impl_.getInternalListenerManager(); - } - TimerPtr createTimer(TimerCb cb) override { return impl_.createTimer(std::move(cb)); } TimerPtr createScaledTimer(ScaledTimerMinimum minimum, TimerCb cb) override { return impl_.createScaledTimer(minimum, std::move(cb)); From 156415174d3ca4e0232e0ed14feaf821caf1e596 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 11 Jun 2021 21:17:41 +0000 Subject: [PATCH 07/23] revert internal listener Signed-off-by: Yuchen Dai --- envoy/network/listener.h | 45 --- source/common/network/BUILD | 1 - source/common/network/connection_impl.cc | 25 +- source/common/network/connection_impl.h | 1 + source/common/network/socket_impl.cc | 8 - source/common/network/utility.cc | 3 - source/common/runtime/runtime_features.cc | 1 - source/common/tcp_proxy/tcp_proxy.cc | 3 - source/common/tcp_proxy/tcp_proxy.h | 2 +- source/common/upstream/upstream_impl.cc | 4 +- source/extensions/io_socket/user_space/BUILD | 1 - .../io_socket/user_space/io_handle_impl.cc | 14 +- .../io_socket/user_space/io_handle_impl.h | 2 - source/server/BUILD | 31 -- source/server/active_internal_listener.cc | 178 ---------- source/server/active_internal_listener.h | 150 --------- source/server/admin/admin.h | 3 - source/server/connection_handler_impl.cc | 42 +-- source/server/connection_handler_impl.h | 12 +- source/server/listener_impl.cc | 22 +- source/server/listener_impl.h | 17 +- source/server/listener_manager_impl.cc | 12 +- test/common/network/BUILD | 1 - test/common/network/connection_impl_test.cc | 38 --- test/config/utility.cc | 13 +- .../proxy_protocol_regression_test.cc | 3 - .../proxy_protocol/proxy_protocol_test.cc | 6 - .../user_space/io_handle_impl_test.cc | 9 - test/integration/BUILD | 1 - test/integration/base_integration_test.h | 7 - test/integration/fake_upstream.h | 4 - test/integration/integration_tcp_client.cc | 1 + test/integration/server.cc | 10 - test/integration/server.h | 10 - .../socket_interface_integration_test.cc | 5 +- test/mocks/network/mocks.h | 1 - test/mocks/server/worker.cc | 4 +- test/server/BUILD | 83 ----- test/server/active_internal_listener_test.cc | 187 ----------- test/server/connection_handler_test.cc | 138 +------- test/server/listener_manager_impl_test.cc | 311 ------------------ 41 files changed, 31 insertions(+), 1378 deletions(-) delete mode 100644 source/server/active_internal_listener.cc delete mode 100644 source/server/active_internal_listener.h delete mode 100644 test/server/active_internal_listener_test.cc diff --git a/envoy/network/listener.h b/envoy/network/listener.h index 5e4ba602fb78f..d4377278d59a2 100644 --- a/envoy/network/listener.h +++ b/envoy/network/listener.h @@ -90,16 +90,6 @@ class UdpListenerConfig { using UdpListenerConfigOptRef = OptRef; -/** - * Configuration for an internal listener. - */ -class InternalListenerConfig { -public: - virtual ~InternalListenerConfig() = default; -}; - -using InternalListenerConfigOptRef = OptRef; - /** * A configuration for an individual listener. */ @@ -178,11 +168,6 @@ class ListenerConfig { */ virtual UdpListenerConfigOptRef udpListenerConfig() PURE; - /** - * @return the internal configuration for the listener IFF it is an internal listener. - */ - virtual InternalListenerConfigOptRef internalListenerConfig() PURE; - /** * @return traffic direction of the listener. */ @@ -426,36 +411,6 @@ class UdpListener : public virtual Listener { using UdpListenerPtr = std::unique_ptr; -class InternalListenerCallbacks { -public: - virtual ~InternalListenerCallbacks() = default; - - /** - * Called when a new connection is accepted. - * @param socket supplies the socket that is moved into the callee. - */ - virtual void onAccept(ConnectionSocketPtr&& socket) PURE; - - virtual Event::Dispatcher& dispatcher() PURE; -}; -using InternalListenerCallbacksOptRef = - absl::optional>; - -class InternalListener {}; - -using InternalListenerPtr = std::unique_ptr; -using InternalListenerOptRef = absl::optional>; - -class InternalListenerManager { -public: - virtual ~InternalListenerManager() = default; - virtual InternalListenerCallbacksOptRef - findByAddress(const Address::InstanceConstSharedPtr& listen_address) PURE; -}; - -using InternalListenerManagerOptRef = - absl::optional>; - /** * Handles delivering datagrams to the correct worker. */ diff --git a/source/common/network/BUILD b/source/common/network/BUILD index 253e713911419..650edfa192ea4 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -62,7 +62,6 @@ envoy_cc_library( hdrs = ["connection_impl_base.h"], deps = [ ":filter_manager_lib", - ":listen_socket_lib", "//envoy/common:scope_tracker_interface", "//envoy/event:dispatcher_interface", "//source/common/common:assert_lib", diff --git a/source/common/network/connection_impl.cc b/source/common/network/connection_impl.cc index e0a067f9a9052..9f37d4b5b70da 100644 --- a/source/common/network/connection_impl.cc +++ b/source/common/network/connection_impl.cc @@ -21,7 +21,6 @@ #include "source/common/network/listen_socket_impl.h" #include "source/common/network/raw_buffer_socket.h" #include "source/common/network/utility.h" -#include "source/common/runtime/runtime_features.h" namespace Envoy { namespace Network { @@ -275,8 +274,7 @@ void ConnectionImpl::noDelay(bool enable) { } // Don't set NODELAY for unix domain sockets - if (socket_->addressType() == Address::Type::Pipe || - socket_->addressType() == Address::Type::EnvoyInternal) { + if (socket_->addressType() == Address::Type::Pipe) { return; } @@ -659,16 +657,9 @@ void ConnectionImpl::onWriteReady() { if (connecting_) { int error; - if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.internal_address") && - socket_->addressProvider().remoteAddress()->type() == - Network::Address::Type::EnvoyInternal) { - // Connected! - error = 0; - } else { - socklen_t error_size = sizeof(error); - RELEASE_ASSERT(socket_->getSocketOption(SOL_SOCKET, SO_ERROR, &error, &error_size).rc_ == 0, - ""); - } + socklen_t error_size = sizeof(error); + RELEASE_ASSERT(socket_->getSocketOption(SOL_SOCKET, SO_ERROR, &error, &error_size).rc_ == 0, + ""); if (error == 0) { ENVOY_CONN_LOG(debug, "connected", *this); @@ -869,12 +860,8 @@ void ClientConnectionImpl::connect() { socket_->addressProvider().remoteAddress()->asString()); const Api::SysCallIntResult result = socket_->connect(socket_->addressProvider().remoteAddress()); if (result.rc_ == 0) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.internal_address") && - socket_->addressProvider().remoteAddress()->type() != - Network::Address::Type::EnvoyInternal) { - // write will become ready. - ASSERT(connecting_); - } + // write will become ready. + ASSERT(connecting_); } else { ASSERT(SOCKET_FAILURE(result.rc_)); #ifdef WIN32 diff --git a/source/common/network/connection_impl.h b/source/common/network/connection_impl.h index a8ffea1f1b25e..4e9f0044924c1 100644 --- a/source/common/network/connection_impl.h +++ b/source/common/network/connection_impl.h @@ -239,6 +239,7 @@ class ClientConnectionImpl : public ConnectionImpl, virtual public ClientConnect const Address::InstanceConstSharedPtr& source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options); + // Network::ClientConnection void connect() override; diff --git a/source/common/network/socket_impl.cc b/source/common/network/socket_impl.cc index 27b7d7beeb8a0..08505cb65b0a4 100644 --- a/source/common/network/socket_impl.cc +++ b/source/common/network/socket_impl.cc @@ -23,18 +23,10 @@ SocketImpl::SocketImpl(IoHandlePtr&& io_handle, address_provider_(std::make_shared(local_address, remote_address)) { if (address_provider_->localAddress() != nullptr) { - // The remote address should have the exact local address type. - ASSERT(address_provider_->remoteAddress() == nullptr || - address_provider_->remoteAddress()->type() == address_provider_->localAddress()->type()); addr_type_ = address_provider_->localAddress()->type(); return; } - if (address_provider_->remoteAddress() != nullptr) { - addr_type_ = address_provider_->remoteAddress()->type(); - return; - } - // Should not happen but some tests inject -1 fds if (!io_handle_->isOpen()) { return; diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index e6d1056ea87c0..df7c966232df1 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -523,9 +523,6 @@ Utility::protobufAddressSocketType(const envoy::config::core::v3::Address& proto } case envoy::config::core::v3::Address::AddressCase::kPipe: return Socket::Type::Stream; - case envoy::config::core::v3::Address::AddressCase::kEnvoyInternalAddress: - // Currently internal address supports stream operation only. - return Socket::Type::Stream; default: NOT_REACHED_GCOVR_EXCL_LINE; } diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 81fcd663832d3..b1aa82b623822 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -74,7 +74,6 @@ constexpr const char* runtime_features[] = { "envoy.reloadable_features.http_upstream_wait_connect_response", "envoy.reloadable_features.http2_skip_encoding_empty_trailers", "envoy.reloadable_features.improved_stream_limit_handling", - "envoy.reloadable_features.internal_address", "envoy.reloadable_features.internal_redirects_with_body", "envoy.reloadable_features.new_tcp_connection_pool", "envoy.reloadable_features.prefer_quic_kernel_bpf_packet_routing", diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index 8eb63a810878f..37f71070dd9d2 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -589,8 +589,6 @@ Network::FilterStatus Filter::onNewConnection() { } void Filter::onDownstreamEvent(Network::ConnectionEvent event) { - ENVOY_CONN_LOG(trace, "on downstream event {}, has upstream = {}", read_callbacks_->connection(), - static_cast(event), upstream_ == nullptr); if (upstream_) { Tcp::ConnectionPool::ConnectionDataPtr conn_data(upstream_->onDownstreamEvent(event)); if (conn_data != nullptr && @@ -751,7 +749,6 @@ Drainer::Drainer(UpstreamDrainManager& parent, const Config::SharedConfigSharedP const Upstream::HostDescriptionConstSharedPtr& upstream_host) : parent_(parent), callbacks_(callbacks), upstream_conn_data_(std::move(conn_data)), timer_(std::move(idle_timer)), upstream_host_(upstream_host), config_(config) { - ENVOY_CONN_LOG(debug, "draining the upstream connection", upstream_conn_data_->connection()); config_->stats().upstream_flush_total_.inc(); config_->stats().upstream_flush_active_.inc(); } diff --git a/source/common/tcp_proxy/tcp_proxy.h b/source/common/tcp_proxy/tcp_proxy.h index eb88a2a84005c..e51d3e7ffbcdf 100644 --- a/source/common/tcp_proxy/tcp_proxy.h +++ b/source/common/tcp_proxy/tcp_proxy.h @@ -389,7 +389,7 @@ class Filter : public Network::ReadFilter, // This class deals with an upstream connection that needs to finish flushing, when the downstream // connection has been closed. The TcpProxy is destroyed when the downstream connection is closed, // so handling the upstream connection here allows it to finish draining or timeout. -class Drainer : public Event::DeferredDeletable, protected Logger::Loggable { +class Drainer : public Event::DeferredDeletable { public: Drainer(UpstreamDrainManager& parent, const Config::SharedConfigSharedPtr& config, const std::shared_ptr& callbacks, diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 23a255af9796e..d710972e00311 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -336,9 +336,7 @@ HostImpl::createConnection(Event::Dispatcher& dispatcher, const ClusterInfo& clu } else { connection_options = options; } - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.internal_address")) { - ASSERT(!address->envoyInternalAddress()); - } + ASSERT(!address->envoyInternalAddress()); Network::ClientConnectionPtr connection = dispatcher.createClientConnection( address, cluster.sourceAddress(), socket_factory.createTransportSocket(std::move(transport_socket_options)), diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 01a5948f4c9e4..334fe51a916e4 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -13,7 +13,6 @@ envoy_cc_extension( name = "config", srcs = ["config.h"], deps = [ - ":io_handle_impl_lib", ], ) diff --git a/source/extensions/io_socket/user_space/io_handle_impl.cc b/source/extensions/io_socket/user_space/io_handle_impl.cc index 25b7222292dca..bbdb37e3b09ef 100644 --- a/source/extensions/io_socket/user_space/io_handle_impl.cc +++ b/source/extensions/io_socket/user_space/io_handle_impl.cc @@ -274,16 +274,10 @@ Network::IoHandlePtr IoHandleImpl::accept(struct sockaddr*, socklen_t*) { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } -Api::SysCallIntResult IoHandleImpl::connect(Network::Address::InstanceConstSharedPtr address) { - if (peer_handle_ != nullptr) { - // Buffered Io handle should always be considered as connected unless the server peer cannot be - // found. Use write or read to determine if peer is closed. - return {0, 0}; - } else { - ENVOY_LOG(debug, "user namespace handle {} connect to {} the closed peer.", - static_cast(this), address->asStringView()); - return Api::SysCallIntResult{-1, SOCKET_ERROR_INVAL}; - } +Api::SysCallIntResult IoHandleImpl::connect(Network::Address::InstanceConstSharedPtr) { + // Buffered Io handle should always be considered as connected. + // Use write or read to determine if peer is closed. + return {0, 0}; } Api::SysCallIntResult IoHandleImpl::setOption(int, int, const void*, socklen_t) { diff --git a/source/extensions/io_socket/user_space/io_handle_impl.h b/source/extensions/io_socket/user_space/io_handle_impl.h index 71d2161be17f3..71d3248d47223 100644 --- a/source/extensions/io_socket/user_space/io_handle_impl.h +++ b/source/extensions/io_socket/user_space/io_handle_impl.h @@ -143,8 +143,6 @@ class IoHandleImpl final : public Network::IoHandle, ASSERT(!peer_handle_); ASSERT(!write_shutdown_); peer_handle_ = writable_peer; - ENVOY_LOG(trace, "io handle {} set peer handle to {}.", static_cast(this), - static_cast(writable_peer)); } private: diff --git a/source/server/BUILD b/source/server/BUILD index 05cc42a02ecaf..28947666d5c61 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -64,7 +64,6 @@ envoy_cc_library( envoy_cc_library( name = "connection_handler_lib", deps = [ - ":active_internal_listener", ":active_tcp_listener", ":active_udp_listener", ":connection_handler_impl", @@ -78,7 +77,6 @@ envoy_cc_library( "connection_handler_impl.h", ], deps = [ - ":active_internal_listener", ":active_tcp_listener", "//envoy/common:time_interface", "//envoy/event:deferred_deletable", @@ -189,35 +187,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "active_internal_listener", - srcs = ["active_internal_listener.cc"], - hdrs = [ - "active_internal_listener.h", - ], - deps = [ - ":active_stream_socket", - "//envoy/common:time_interface", - "//envoy/event:deferred_deletable", - "//envoy/event:dispatcher_interface", - "//envoy/event:timer_interface", - "//envoy/network:connection_handler_interface", - "//envoy/network:connection_interface", - "//envoy/network:exception_interface", - "//envoy/network:filter_interface", - "//envoy/network:listen_socket_interface", - "//envoy/network:listener_interface", - "//envoy/server:listener_manager_interface", - "//envoy/stats:timespan_interface", - "//source/common/common:linked_object", - "//source/common/common:non_copyable", - "//source/common/event:deferred_task", - "//source/common/network:connection_lib", - "//source/common/stats:timespan_lib", - "//source/common/stream_info:stream_info_lib", - ], -) - envoy_cc_library( name = "drain_manager_lib", srcs = ["drain_manager_impl.cc"], diff --git a/source/server/active_internal_listener.cc b/source/server/active_internal_listener.cc deleted file mode 100644 index 0bc822f691c85..0000000000000 --- a/source/server/active_internal_listener.cc +++ /dev/null @@ -1,178 +0,0 @@ -#include "source/server/active_internal_listener.h" - -#include "envoy/network/filter.h" -#include "envoy/stats/scope.h" - -#include "source/common/network/address_impl.h" -#include "source/common/stats/timespan_impl.h" - -namespace Envoy { -namespace Server { - -ActiveInternalListener::ActiveInternalListener(Network::ConnectionHandler& conn_handler, - Event::Dispatcher& dispatcher, - Network::ListenerConfig& config) - : TypedActiveStreamListenerBase( - conn_handler, dispatcher, - std::make_unique(), config) {} -ActiveInternalListener::ActiveInternalListener(Network::ConnectionHandler& conn_handler, - Event::Dispatcher& dispatcher, - Network::ListenerPtr listener, - Network::ListenerConfig& config) - : TypedActiveStreamListenerBase(conn_handler, dispatcher, - std::move(listener), config) {} - -ActiveInternalListener::~ActiveInternalListener() { cleanupConnections(); } - -void ActiveInternalListener::removeConnection(ActiveInternalConnection& connection) { - ENVOY_CONN_LOG(debug, "adding to cleanup list", *connection.connection_); - ActiveInternalConnections& active_connections = connection.active_connections_; - ActiveInternalConnectionPtr removed = connection.removeFromList(active_connections.connections_); - dispatcher().deferredDelete(std::move(removed)); - // Delete map entry only iff connections becomes empty. - if (active_connections.connections_.empty()) { - auto iter = connections_by_context_.find(&active_connections.filter_chain_); - ASSERT(iter != connections_by_context_.end()); - // To cover the lifetime of every single connection, Connections need to be deferred deleted - // because the previously contained connection is deferred deleted. - dispatcher().deferredDelete(std::move(iter->second)); - // The erase will break the iteration over the connections_by_context_ during the deletion. - if (!is_deleting_) { - connections_by_context_.erase(iter); - } - } -} - -void ActiveInternalListener::newConnection(Network::ConnectionSocketPtr&& socket, - std::unique_ptr stream_info) { - - // Find matching filter chain. - const auto filter_chain = config_->filterChainManager().findFilterChain(*socket); - if (filter_chain == nullptr) { - ENVOY_LOG(debug, "closing connection: no matching filter chain found"); - stats_.no_filter_chain_match_.inc(); - stream_info->setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); - stream_info->setResponseCodeDetails(StreamInfo::ResponseCodeDetails::get().FilterChainNotFound); - emitLogs(*config_, *stream_info); - socket->close(); - return; - } - - stream_info->setFilterChainName(filter_chain->name()); - auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); - stream_info->setDownstreamSslConnection(transport_socket->ssl()); - auto& active_connections = getOrCreateActiveConnections(*filter_chain); - auto server_conn_ptr = dispatcher().createServerConnection( - std::move(socket), std::move(transport_socket), *stream_info); - if (const auto timeout = filter_chain->transportSocketConnectTimeout(); - timeout != std::chrono::milliseconds::zero()) { - server_conn_ptr->setTransportSocketConnectTimeout(timeout); - } - ActiveInternalConnectionPtr active_connection( - new ActiveInternalConnection(active_connections, std::move(server_conn_ptr), - dispatcher().timeSource(), std::move(stream_info))); - active_connection->connection_->setBufferLimits(config_->perConnectionBufferLimitBytes()); - - const bool empty_filter_chain = !config_->filterChainFactory().createNetworkFilterChain( - *active_connection->connection_, filter_chain->networkFilterFactories()); - if (empty_filter_chain) { - ENVOY_CONN_LOG(debug, "closing connection: no filters", *active_connection->connection_); - active_connection->connection_->close(Network::ConnectionCloseType::NoFlush); - } - - // If the connection is already closed, we can just let this connection immediately die. - if (active_connection->connection_->state() != Network::Connection::State::Closed) { - ENVOY_CONN_LOG(debug, "new connection", *active_connection->connection_); - active_connection->connection_->addConnectionCallbacks(*active_connection); - LinkedList::moveIntoList(std::move(active_connection), active_connections.connections_); - } -} - -ActiveInternalConnections& -ActiveInternalListener::getOrCreateActiveConnections(const Network::FilterChain& filter_chain) { - ActiveInternalConnectionsPtr& connections = connections_by_context_[&filter_chain]; - if (connections == nullptr) { - connections = std::make_unique(*this, filter_chain); - } - return *connections; -} - -void ActiveInternalListener::updateListenerConfig(Network::ListenerConfig& config) { - ENVOY_LOG(trace, "replacing listener ", config_->listenerTag(), " by ", config.listenerTag()); - config_ = &config; -} - -ActiveInternalConnections::ActiveInternalConnections(ActiveInternalListener& listener, - const Network::FilterChain& filter_chain) - : listener_(listener), filter_chain_(filter_chain) {} - -ActiveInternalConnections::~ActiveInternalConnections() { - // connections should be defer deleted already. - ASSERT(connections_.empty()); -} - -ActiveInternalConnection::ActiveInternalConnection( - ActiveInternalConnections& active_connections, Network::ConnectionPtr&& new_connection, - TimeSource& time_source, std::unique_ptr&& stream_info) - : stream_info_(std::move(stream_info)), active_connections_(active_connections), - connection_(std::move(new_connection)), - conn_length_(new Stats::HistogramCompletableTimespanImpl( - active_connections_.listener_.stats_.downstream_cx_length_ms_, time_source)) { - // We just universally set no delay on connections. Theoretically we might at some point want - // to make this configurable. - connection_->noDelay(true); - auto& listener = active_connections_.listener_; - listener.stats_.downstream_cx_total_.inc(); - listener.stats_.downstream_cx_active_.inc(); - listener.per_worker_stats_.downstream_cx_total_.inc(); - listener.per_worker_stats_.downstream_cx_active_.inc(); - stream_info_->setConnectionID(connection_->id()); - - // Active connections on the handler (not listener). The per listener connections have already - // been incremented at this point either via the connection balancer or in the socket accept - // path if there is no configured balancer. - listener.parent_.incNumConnections(); -} - -ActiveInternalConnection::~ActiveInternalConnection() { - ActiveInternalListener::emitLogs(*active_connections_.listener_.config_, *stream_info_); - auto& listener = active_connections_.listener_; - listener.stats_.downstream_cx_active_.dec(); - listener.stats_.downstream_cx_destroy_.inc(); - listener.per_worker_stats_.downstream_cx_active_.dec(); - conn_length_->complete(); - - // Active listener connections (not handler). - listener.decNumConnections(); - - // Active handler connections (not listener). - listener.parent_.decNumConnections(); -} - -void ActiveInternalListener::onAccept(Network::ConnectionSocketPtr&& socket) { - // Unlike tcp listener, no rebalancer is applied and won't call pickTargetHandler to account - // connections. - incNumConnections(); - - auto active_socket = std::make_unique( - *this, std::move(socket), false /* do not handle off at internal listener */); - // TODO(lambdai): restore address from either socket options, or from listener config. - active_socket->socket_->addressProvider().restoreLocalAddress( - std::make_shared("255.255.255.255", 0)); - active_socket->socket_->addressProvider().setRemoteAddress( - std::make_shared("255.255.255.254", 0)); - - onSocketAccepted(std::move(active_socket)); -} - -// Network::ConnectionCallbacks -void ActiveInternalConnection::onEvent(Network::ConnectionEvent event) { - FANCY_LOG(info, "lambdai: internal connection on event {}", static_cast(event)); - // Any event leads to destruction of the connection. - if (event == Network::ConnectionEvent::LocalClose || - event == Network::ConnectionEvent::RemoteClose) { - active_connections_.listener_.removeConnection(*this); - } -} -} // namespace Server -} // namespace Envoy diff --git a/source/server/active_internal_listener.h b/source/server/active_internal_listener.h deleted file mode 100644 index eaf12026bb9bd..0000000000000 --- a/source/server/active_internal_listener.h +++ /dev/null @@ -1,150 +0,0 @@ -#pragma once -#include -#include -#include -#include - -#include "envoy/common/time.h" -#include "envoy/event/deferred_deletable.h" -#include "envoy/event/dispatcher.h" -#include "envoy/network/connection.h" -#include "envoy/network/connection_handler.h" -#include "envoy/network/filter.h" -#include "envoy/network/listen_socket.h" -#include "envoy/network/listener.h" -#include "envoy/server/listener_manager.h" -#include "envoy/stats/scope.h" -#include "envoy/stats/timespan.h" - -#include "source/common/common/linked_object.h" -#include "source/common/common/non_copyable.h" -#include "source/common/stream_info/stream_info_impl.h" -#include "source/server/active_stream_socket.h" - -#include "spdlog/spdlog.h" - -namespace Envoy { -namespace Server { - -struct ActiveInternalConnection; -using ActiveInternalConnectionPtr = std::unique_ptr; - -class ActiveInternalConnections; -using ActiveInternalConnectionsPtr = std::unique_ptr; - -class ActiveInternalListener; -/** - * Wrapper for a group of active connections which are attached to the same filter chain context. - */ -class ActiveInternalConnections : public Event::DeferredDeletable { -public: - ActiveInternalConnections(ActiveInternalListener& listener, - const Network::FilterChain& filter_chain); - ~ActiveInternalConnections() override; - - // Listener filter chain pair is the owner of the connections. - ActiveInternalListener& listener_; - const Network::FilterChain& filter_chain_; - // Owned connections. - std::list connections_; -}; - -/** - * Wrapper for an active internal connection owned by this handler. - */ -struct ActiveInternalConnection : LinkedObject, - public Event::DeferredDeletable, - public Network::ConnectionCallbacks { - ActiveInternalConnection(ActiveInternalConnections& active_connections, - Network::ConnectionPtr&& new_connection, TimeSource& time_system, - std::unique_ptr&& stream_info); - ~ActiveInternalConnection() override; - - using CollectionType = ActiveInternalConnections; - - // Network::ConnectionCallbacks - void onEvent(Network::ConnectionEvent event) override; - void onAboveWriteBufferHighWatermark() override {} - void onBelowWriteBufferLowWatermark() override {} - - std::unique_ptr stream_info_; - ActiveInternalConnections& active_connections_; - Network::ConnectionPtr connection_; - Stats::TimespanPtr conn_length_; -}; - -class ActiveInternalListener : public TypedActiveStreamListenerBase, - public Network::InternalListenerCallbacks, - Logger::Loggable { -public: - ActiveInternalListener(Network::ConnectionHandler& conn_handler, Event::Dispatcher& dispatcher, - Network::ListenerConfig& config); - ActiveInternalListener(Network::ConnectionHandler& conn_handler, Event::Dispatcher& dispatcher, - Network::ListenerPtr listener, Network::ListenerConfig& config); - ~ActiveInternalListener() override; - - class NetworkInternalListener : public Network::Listener { - - void disable() override { - // TODO(lambdai): think about how to elegantly disable internal listener. (Queue socket or - // close socket immediately?) - ENVOY_LOG(debug, "Warning: the internal listener cannot be disabled."); - } - - void enable() override { - ENVOY_LOG(debug, "Warning: the internal listener is always enabled."); - } - - void setRejectFraction(UnitFloat) override {} - }; - // ActiveListenerImplBase - Network::Listener* listener() override { return listener_.get(); } - Network::BalancedConnectionHandlerOptRef - getBalancedHandlerByAddress(const Network::Address::Instance&) override { - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; - } - - void pauseListening() override { - if (listener_ != nullptr) { - listener_->disable(); - } - } - void resumeListening() override { - if (listener_ != nullptr) { - listener_->enable(); - } - } - void shutdownListener() override { listener_.reset(); } - - // Network::InternalListenerCallbacks - void onAccept(Network::ConnectionSocketPtr&& socket) override; - Event::Dispatcher& dispatcher() override { return dispatcher_; } - - void incNumConnections() override { config_->openConnections().inc(); } - void decNumConnections() override { config_->openConnections().dec(); } - /** - * Remove and destroy an active connection. - * @param connection supplies the connection to remove. - */ - void removeConnection(ActiveInternalConnection& connection); - - /** - * Create a new connection from a socket accepted by the listener. - */ - void newConnection(Network::ConnectionSocketPtr&& socket, - std::unique_ptr stream_info) override; - - /** - * Return the active connections container attached with the given filter chain. - */ - ActiveInternalConnections& getOrCreateActiveConnections(const Network::FilterChain& filter_chain); - - /** - * Update the listener config. The follow up connections will see the new config. The existing - * connections are not impacted. - */ - void updateListenerConfig(Network::ListenerConfig& config); -}; - -} // namespace Server -} // namespace Envoy diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index a14715934cbcb..5fcaa5ddcf603 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -359,9 +359,6 @@ class AdminImpl : public Admin, Network::UdpListenerConfigOptRef udpListenerConfig() override { return Network::UdpListenerConfigOptRef(); } - Network::InternalListenerConfigOptRef internalListenerConfig() override { - return Network::InternalListenerConfigOptRef(); - } envoy::config::core::v3::TrafficDirection direction() const override { return envoy::config::core::v3::UNSPECIFIED; } diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index 4a142fea3a903..79a6710d205b3 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -5,10 +5,8 @@ #include "envoy/event/dispatcher.h" #include "envoy/network/filter.h" -#include "source/common/common/logger.h" #include "source/common/event/deferred_task.h" #include "source/common/network/utility.h" -#include "source/server/active_internal_listener.h" #include "source/server/active_tcp_listener.h" namespace Envoy { @@ -29,20 +27,7 @@ void ConnectionHandlerImpl::decNumConnections() { void ConnectionHandlerImpl::addListener(absl::optional overridden_listener, Network::ListenerConfig& config) { ActiveListenerDetails details; - if (config.internalListenerConfig().has_value()) { - if (overridden_listener.has_value()) { - for (auto& listener : listeners_) { - if (listener.second.listener_->listenerTag() == overridden_listener) { - listener.second.internalListener()->get().updateListenerConfig(config); - return; - } - } - NOT_REACHED_GCOVR_EXCL_LINE; - } - auto internal_listener = std::make_unique(*this, dispatcher(), config); - details.typed_listener_ = *internal_listener; - details.listener_ = std::move(internal_listener); - } else if (config.listenSocketFactory().socketType() == Network::Socket::Type::Stream) { + if (config.listenSocketFactory().socketType() == Network::Socket::Type::Stream) { if (overridden_listener.has_value()) { for (auto& listener : listeners_) { if (listener.second.listener_->listenerTag() == overridden_listener) { @@ -146,25 +131,6 @@ void ConnectionHandlerImpl::setListenerRejectFraction(UnitFloat reject_fraction) } } -Network::InternalListenerCallbacksOptRef -ConnectionHandlerImpl::findByAddress(const Network::Address::InstanceConstSharedPtr& address) { - auto listener_it = - std::find_if(listeners_.begin(), listeners_.end(), - [&address](std::pair& p) { - return p.second.internalListener().has_value() && - p.second.listener_->listener() != nullptr && - p.first->type() == Network::Address::Type::EnvoyInternal && - *(p.first) == *address; - }); - - if (listener_it != listeners_.end()) { - return Network::InternalListenerCallbacksOptRef( - listener_it->second.internalListener().value().get()); - } - return absl::nullopt; -} - ConnectionHandlerImpl::ActiveTcpListenerOptRef ConnectionHandlerImpl::ActiveListenerDetails::tcpListener() { auto* val = absl::get_if>(&typed_listener_); @@ -177,12 +143,6 @@ ConnectionHandlerImpl::ActiveListenerDetails::udpListener() { return (val != nullptr) ? absl::make_optional(*val) : absl::nullopt; } -ConnectionHandlerImpl::ActiveInternalListenerOptRef -ConnectionHandlerImpl::ActiveListenerDetails::internalListener() { - auto* val = absl::get_if>(&typed_listener_); - return (val != nullptr) ? absl::make_optional(*val) : absl::nullopt; -} - ConnectionHandlerImpl::ActiveListenerDetailsOptRef ConnectionHandlerImpl::findActiveListenerByTag(uint64_t listener_tag) { // TODO(mattklein123): We should probably use a hash table here to lookup the tag diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index c58543a1d8fc4..f10424b7c1a2d 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -21,7 +21,6 @@ namespace Server { class ActiveUdpListenerBase; class ActiveTcpListener; -class ActiveInternalListener; /** * Server side connection handler. This is used both by workers as well as the @@ -29,15 +28,12 @@ class ActiveInternalListener; */ class ConnectionHandlerImpl : public Network::TcpConnectionHandler, public Network::UdpConnectionHandler, - public Network::InternalListenerManager, NonCopyable, Logger::Loggable { public: using UdpListenerCallbacksOptRef = absl::optional>; using ActiveTcpListenerOptRef = absl::optional>; - using ActiveInternalListenerOptRef = - absl::optional>; ConnectionHandlerImpl(Event::Dispatcher& dispatcher, absl::optional worker_index); @@ -67,24 +63,18 @@ class ConnectionHandlerImpl : public Network::TcpConnectionHandler, // Network::UdpConnectionHandler Network::UdpListenerCallbacksOptRef getUdpListenerCallbacks(uint64_t listener_tag) override; - // Network::InternalListenerManager - Network::InternalListenerCallbacksOptRef - findByAddress(const Network::Address::InstanceConstSharedPtr& listen_address) override; - private: struct ActiveListenerDetails { // Strong pointer to the listener, whether TCP, UDP, QUIC, etc. Network::ConnectionHandler::ActiveListenerPtr listener_; absl::variant, - std::reference_wrapper, - std::reference_wrapper> + std::reference_wrapper> typed_listener_; // Helpers for accessing the data in the variant for cleaner code. ActiveTcpListenerOptRef tcpListener(); UdpListenerCallbacksOptRef udpListener(); - ActiveInternalListenerOptRef internalListener(); }; using ActiveListenerDetailsOptRef = absl::optional>; ActiveListenerDetailsOptRef findActiveListenerByTag(uint64_t listener_tag); diff --git a/source/server/listener_impl.cc b/source/server/listener_impl.cc index f0643316cc10c..3192d28392502 100644 --- a/source/server/listener_impl.cc +++ b/source/server/listener_impl.cc @@ -92,8 +92,7 @@ ListenSocketFactoryImpl::ListenSocketFactoryImpl(ListenerComponentFactory& facto bind_to_port_(bind_to_port), listener_name_(listener_name), reuse_port_(reuse_port) { bool create_socket = false; - switch (local_address_->type()) { - case Network::Address::Type::Ip: + if (local_address_->type() == Network::Address::Type::Ip) { if (socket_type_ == Network::Socket::Type::Datagram) { ASSERT(reuse_port_ == true); } @@ -106,14 +105,10 @@ ListenSocketFactoryImpl::ListenSocketFactoryImpl(ListenerComponentFactory& facto // then all worker threads should use same port. create_socket = true; } - break; - case Network::Address::Type::Pipe: + } else { + ASSERT(local_address_->type() == Network::Address::Type::Pipe); // Listeners with Unix domain socket always use shared socket. create_socket = true; - break; - case Network::Address::Type::EnvoyInternal: - create_socket = false; - break; } if (create_socket) { @@ -329,7 +324,6 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, buildOriginalDstListenerFilter(); buildProxyProtocolListenerFilter(); buildTlsInspectorListenerFilter(); - buildInternalListener(); } if (!workers_started_) { // Initialize dynamic_init_manager_ from Server's init manager if it's not initialized. @@ -357,8 +351,7 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, validation_visitor_( added_via_api_ ? parent_.server_.messageValidationContext().dynamicValidationVisitor() : parent_.server_.messageValidationContext().staticValidationVisitor()), - // listener_init_target_ is not used during in place update because we expect server - // started. + // listener_init_target_ is not used during in place update because we expect server started. listener_init_target_("", nullptr), dynamic_init_manager_(std::make_unique( fmt::format("Listener-local-init-manager {} {}", name, hash))), @@ -393,7 +386,6 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, buildOriginalDstListenerFilter(); buildProxyProtocolListenerFilter(); buildTlsInspectorListenerFilter(); - buildInternalListener(); open_connections_ = origin.open_connections_; } @@ -405,12 +397,6 @@ void ListenerImpl::buildAccessLog() { } } -void ListenerImpl::buildInternalListener() { - if (config_.has_internal_listener()) { - internal_listener_config_ = std::make_unique(); - } -} - void ListenerImpl::buildUdpListenerFactory(Network::Socket::Type socket_type, uint32_t concurrency) { if (socket_type != Network::Socket::Type::Datagram) { diff --git a/source/server/listener_impl.h b/source/server/listener_impl.h index c428bc4e0298a..d04613f030692 100644 --- a/source/server/listener_impl.h +++ b/source/server/listener_impl.h @@ -60,9 +60,7 @@ class ListenSocketFactoryImpl : public Network::ListenSocketFactory, * @return the socket shared by worker threads; otherwise return null. */ Network::SocketOptRef sharedSocket() const override { - if (!reuse_port_ && - // internal address has no listener socket. - local_address_->envoyInternalAddress() == nullptr) { + if (!reuse_port_) { ASSERT(socket_ != nullptr); return *socket_; } @@ -310,10 +308,6 @@ class ListenerImpl final : public Network::ListenerConfig, return udp_listener_config_ != nullptr ? *udp_listener_config_ : Network::UdpListenerConfigOptRef(); } - Network::InternalListenerConfigOptRef internalListenerConfig() override { - return internal_listener_config_ != nullptr ? *internal_listener_config_ - : Network::InternalListenerConfigOptRef(); - } Network::ConnectionBalancer& connectionBalancer() override { return *connection_balancer_; } ResourceLimit& openConnections() override { return *open_connections_; } const std::vector& accessLogs() const override { @@ -360,13 +354,6 @@ class ListenerImpl final : public Network::ListenerConfig, Network::UdpListenerWorkerRouterPtr listener_worker_router_; }; - struct InternalListenerConfigImpl : public Network::InternalListenerConfig { - InternalListenerConfigImpl( - const envoy::config::listener::v3::Listener_InternalListenerConfig config) - : config_(config) {} - const envoy::config::listener::v3::Listener_InternalListenerConfig config_; - }; - /** * Create a new listener from an existing listener and the new config message if the in place * filter chain update is decided. Should be called only by newListenerWithFilterChain(). @@ -377,7 +364,6 @@ class ListenerImpl final : public Network::ListenerConfig, uint32_t concurrency); // Helpers for constructor. void buildAccessLog(); - void buildInternalListener(); void buildUdpListenerFactory(Network::Socket::Type socket_type, uint32_t concurrency); void buildListenSocketOptions(Network::Socket::Type socket_type); void createListenerFilterFactories(Network::Socket::Type socket_type); @@ -425,7 +411,6 @@ class ListenerImpl final : public Network::ListenerConfig, const std::chrono::milliseconds listener_filters_timeout_; const bool continue_on_listener_filters_timeout_; std::unique_ptr udp_listener_config_; - std::unique_ptr internal_listener_config_; Network::ConnectionBalancerSharedPtr connection_balancer_; std::shared_ptr listener_factory_context_; FilterChainManagerImpl filter_chain_manager_; diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 4c52682416fe8..1b7921886a70c 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -332,13 +332,11 @@ ListenerManagerStats ListenerManagerImpl::generateStats(Stats::Scope& scope) { bool ListenerManagerImpl::addOrUpdateListener(const envoy::config::listener::v3::Listener& config, const std::string& version_info, bool added_via_api) { - if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.internal_address")) { - RELEASE_ASSERT( - !config.address().has_envoy_internal_address(), - fmt::format("listener {} has envoy internal address {}. Internal address cannot be used by " - "listener yet", - config.name(), config.address().envoy_internal_address().DebugString())); - } + RELEASE_ASSERT( + !config.address().has_envoy_internal_address(), + fmt::format("listener {} has envoy internal address {}. Internal address cannot be used by " + "listener yet", + config.name(), config.address().envoy_internal_address().DebugString())); // TODO(junr03): currently only one ApiListener can be installed via bootstrap to avoid having to // build a collection of listeners, and to have to be able to warm and drain the listeners. In the // future allow multiple ApiListeners, and allow them to be created via LDS as well as bootstrap. diff --git a/test/common/network/BUILD b/test/common/network/BUILD index c0ed193b61f12..ce5f097cd5f5e 100644 --- a/test/common/network/BUILD +++ b/test/common/network/BUILD @@ -91,7 +91,6 @@ envoy_cc_test( "//test/test_common:environment_lib", "//test/test_common:network_utility_lib", "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_runtime_lib", "//test/test_common:test_time_lib", "//test/test_common:threadsafe_singleton_injector_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 4ad1d8d1d3245..63f5a10898aef 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -6,7 +6,6 @@ #include "envoy/config/core/v3/base.pb.h" #include "envoy/event/scaled_range_timer_manager.h" #include "envoy/network/address.h" -#include "envoy/network/listener.h" #include "source/common/api/os_sys_calls_impl.h" #include "source/common/buffer/buffer_impl.h" @@ -30,7 +29,6 @@ #include "test/test_common/network_utility.h" #include "test/test_common/printers.h" #include "test/test_common/simulated_time_system.h" -#include "test/test_common/test_runtime.h" #include "test/test_common/threadsafe_singleton_injector.h" #include "test/test_common/utility.h" @@ -55,12 +53,6 @@ namespace Envoy { namespace Network { namespace { -class MockInternalListenerManager : public InternalListenerManager { -public: - MOCK_METHOD(InternalListenerCallbacksOptRef, findByAddress, - (const Address::InstanceConstSharedPtr&), ()); -}; - TEST(RawBufferSocket, TestBasics) { TransportSocketPtr raw_buffer_socket(Network::Test::createRawBufferSocket()); EXPECT_FALSE(raw_buffer_socket->ssl()); @@ -2958,36 +2950,6 @@ TEST_F(PipeClientConnectionImplTest, SkipSourceAddress) { connection->close(ConnectionCloseType::NoFlush); } -class InternalClientConnectionImplTest : public testing::Test { -protected: - InternalClientConnectionImplTest() - : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")) {} - - Api::ApiPtr api_; - Event::DispatcherPtr dispatcher_; - StrictMock client_callbacks_; -}; - -TEST_F(InternalClientConnectionImplTest, - CannotCreateConnectionToInternalAddressWithInternalAddressEnabled) { - auto scoped_runtime_guard = std::make_unique(); - Runtime::LoaderSingleton::getExisting()->mergeValues( - {{"envoy.reloadable_features.internal_address", "true"}}); - - const Network::SocketInterface* sock_interface = Network::socketInterface( - "envoy.extensions.network.socket_interface.default_socket_interface"); - Network::Address::InstanceConstSharedPtr address = - std::make_shared("listener_0", sock_interface); - // Not implemented yet. - ASSERT_DEATH( - { - ClientConnectionPtr connection = - dispatcher_->createClientConnection(address, Network::Address::InstanceConstSharedPtr(), - Network::Test::createRawBufferSocket(), nullptr); - }, - "panic: not implemented"); -} - } // namespace } // namespace Network } // namespace Envoy diff --git a/test/config/utility.cc b/test/config/utility.cc index 34c7fe2908a96..649b2fc26a2ed 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -600,18 +600,6 @@ ConfigHelper::ConfigHelper(const Network::Address::IpVersion version, Api::Api& auto* static_resources = bootstrap_.mutable_static_resources(); for (int i = 0; i < static_resources->listeners_size(); ++i) { auto* listener = static_resources->mutable_listeners(i); - if (listener->mutable_address()->has_envoy_internal_address()) { - ENVOY_LOG_MISC( - debug, "Listener {} has internal address {}. Will not reset to loop back socket address.", - i, listener->mutable_address()->envoy_internal_address().server_listener_name()); - continue; - } - if (listener->mutable_address()->has_pipe()) { - ENVOY_LOG_MISC(debug, - "Listener {} has pipe address {}. Will not reset to loop back socket address.", - i, listener->mutable_address()->pipe().path()); - continue; - } auto* listener_socket_addr = listener->mutable_address()->mutable_socket_address(); if (listener_socket_addr->address() == "0.0.0.0" || listener_socket_addr->address() == "::") { listener_socket_addr->set_address(Network::Test::getAnyAddressString(version)); @@ -960,6 +948,7 @@ void ConfigHelper::setDefaultHostAndRoute(const std::string& domains, const std: void ConfigHelper::setBufferLimits(uint32_t upstream_buffer_limit, uint32_t downstream_buffer_limit) { RELEASE_ASSERT(!finalized_, ""); + RELEASE_ASSERT(bootstrap_.mutable_static_resources()->listeners_size() == 1, ""); auto* listener = bootstrap_.mutable_static_resources()->mutable_listeners(0); listener->mutable_per_connection_buffer_limit_bytes()->set_value(downstream_buffer_limit); const uint32_t stream_buffer_size = std::max( diff --git a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc index 019aff2f20ee2..f739c187f0da0 100644 --- a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc +++ b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc @@ -72,9 +72,6 @@ class ProxyProtocolRegressionTest : public testing::TestWithParamconnect(address_is_ignored).rc_); } -TEST_F(IoHandleImplTest, ConnectToClosedIoHandle) { - auto address_is_ignored = - std::make_shared("listener_id"); - io_handle_peer_->close(); - auto result = io_handle_->connect(address_is_ignored); - EXPECT_EQ(-1, result.rc_); - EXPECT_EQ(SOCKET_ERROR_INVAL, result.errno_); -} - TEST_F(IoHandleImplTest, ActivateEvent) { schedulable_cb_ = new NiceMock(&dispatcher_); io_handle_->initializeFileEvent( diff --git a/test/integration/BUILD b/test/integration/BUILD index 6168fae8199ac..dbea0c188ff02 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -679,7 +679,6 @@ envoy_cc_test_library( "fake_upstream.h", ], deps = [ - "//source/server:listener_manager_lib", "//envoy/api:api_interface", "//envoy/grpc:status", "//envoy/http:codec_interface", diff --git a/test/integration/base_integration_test.h b/test/integration/base_integration_test.h index e4d081e7c7995..0a76bb8da54e2 100644 --- a/test/integration/base_integration_test.h +++ b/test/integration/base_integration_test.h @@ -360,13 +360,6 @@ class BaseIntegrationTest : protected Logger::Loggable { upstream_config_.http2_options_.MergeFrom(options); } - Event::Dispatcher* getWorkerDispatcher(uint32_t index) { - if (test_server_ == nullptr) { - return nullptr; - } - return &test_server_->getWorkerDispatcher(index).value().get(); - } - std::unique_ptr upstream_stats_store_; // Make sure the test server will be torn down after any fake client. diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index fd5255cabbbbf..580da4db816b3 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -13,7 +13,6 @@ #include "envoy/network/connection.h" #include "envoy/network/connection_handler.h" #include "envoy/network/filter.h" -#include "envoy/network/listener.h" #include "envoy/stats/scope.h" #include "source/common/buffer/buffer_impl.h" @@ -781,9 +780,6 @@ class FakeUpstream : Logger::Loggable, uint64_t listenerTag() const override { return 0; } const std::string& name() const override { return name_; } Network::UdpListenerConfigOptRef udpListenerConfig() override { return udp_listener_config_; } - Network::InternalListenerConfigOptRef internalListenerConfig() override { - return Network::InternalListenerConfigOptRef(); - } Network::ConnectionBalancer& connectionBalancer() override { return connection_balancer_; } envoy::config::core::v3::TrafficDirection direction() const override { return envoy::config::core::v3::UNSPECIFIED; diff --git a/test/integration/integration_tcp_client.cc b/test/integration/integration_tcp_client.cc index 64fa4089ccd16..a6535265c00f3 100644 --- a/test/integration/integration_tcp_client.cc +++ b/test/integration/integration_tcp_client.cc @@ -53,6 +53,7 @@ IntegrationTcpClient::IntegrationTcpClient( std::function above_overflow) -> Buffer::Instance* { return new Buffer::WatermarkBuffer(below_low, above_high, above_overflow); })); + ; connection_ = dispatcher.createClientConnection( Network::Utility::resolveUrl( diff --git a/test/integration/server.cc b/test/integration/server.cc index 90725cc8d6757..99a5ecbd0732b 100644 --- a/test/integration/server.cc +++ b/test/integration/server.cc @@ -241,16 +241,6 @@ void IntegrationTestServerImpl::createAndRunEnvoyServer( admin_address_ = server.admin().socket().addressProvider().localAddress(); server_ = &server; stat_store_ = &stat_store; - // Use the tls slots to save dispatchers. The slot must be destroyed before server shutdown. - ThreadLocal::SlotPtr dispatcher_tls = tls.allocateSlot(); - - dispatcher_tls->set( - [this](Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { - Thread::LockGuard guard(dispatcher_mutex_); - registered_dispatchers_.push_back(dispatcher); - return nullptr; - }); - serverReady(); server.run(); } diff --git a/test/integration/server.h b/test/integration/server.h index 7b2108d050921..a192058fa6e0b 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -505,12 +505,6 @@ class IntegrationTestServer : public Logger::Loggable, void useAdminInterfaceToQuit(bool use) { use_admin_interface_to_quit_ = use; } bool useAdminInterfaceToQuit() { return use_admin_interface_to_quit_; } - absl::optional> getWorkerDispatcher(uint32_t index) { - Thread::LockGuard guard(dispatcher_mutex_); - ASSERT(index < registered_dispatchers_.size()); - return registered_dispatchers_[index]; - } - protected: IntegrationTestServer(Event::TestTimeSystem& time_system, Api::Api& api, const std::string& config_path) @@ -533,10 +527,6 @@ class IntegrationTestServer : public Logger::Loggable, // adminAddress()) will be available. void serverReady(); - Thread::MutexBasicLockable dispatcher_mutex_; - // Assume 128 is greater than concurrency. - std::vector>> registered_dispatchers_; - private: /** * Runs the real server on a thread. diff --git a/test/integration/socket_interface_integration_test.cc b/test/integration/socket_interface_integration_test.cc index d4f78da877888..63a531a415753 100644 --- a/test/integration/socket_interface_integration_test.cc +++ b/test/integration/socket_interface_integration_test.cc @@ -90,6 +90,7 @@ TEST_P(SocketInterfaceIntegrationTest, AddressWithSocketInterface) { } // Test that connecting to internal address will crash. +// TODO(lambdai): Add internal connection implementation to enable the connection creation. TEST_P(SocketInterfaceIntegrationTest, InternalAddressWithSocketInterface) { BaseIntegrationTest::initialize(); @@ -100,8 +101,6 @@ TEST_P(SocketInterfaceIntegrationTest, InternalAddressWithSocketInterface) { Network::Address::InstanceConstSharedPtr address = std::make_shared("listener_0", sock_interface); - Runtime::LoaderSingleton::getExisting()->mergeValues( - {{"envoy.reloadable_features.internal_address", "false"}}); ASSERT_DEATH(client_ = dispatcher_->createClientConnection( address, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr), @@ -109,7 +108,7 @@ TEST_P(SocketInterfaceIntegrationTest, InternalAddressWithSocketInterface) { } // Test that recv from internal address will crash. -// TODO(lambdai): Add UDP internal listener implementation to enable the io path. +// TODO(lambdai): Add internal socket implementation to enable the io path. TEST_P(SocketInterfaceIntegrationTest, UdpRecvFromInternalAddressWithSocketInterface) { BaseIntegrationTest::initialize(); diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 2a84a27b05fc0..e22250f5e214a 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -390,7 +390,6 @@ class MockListenerConfig : public ListenerConfig { MOCK_METHOD(uint64_t, listenerTag, (), (const)); MOCK_METHOD(const std::string&, name, (), (const)); MOCK_METHOD(Network::UdpListenerConfigOptRef, udpListenerConfig, ()); - MOCK_METHOD(InternalListenerConfigOptRef, internalListenerConfig, ()); MOCK_METHOD(ConnectionBalancer&, connectionBalancer, ()); MOCK_METHOD(ResourceLimit&, openConnections, ()); MOCK_METHOD(uint32_t, tcpBacklogSize, (), (const)); diff --git a/test/mocks/server/worker.cc b/test/mocks/server/worker.cc index d7ab0b6f49c42..a7e981b299b10 100644 --- a/test/mocks/server/worker.cc +++ b/test/mocks/server/worker.cc @@ -17,9 +17,7 @@ MockWorker::MockWorker() { Invoke([this](absl::optional overridden_listener, Network::ListenerConfig& config, AddListenerCompletion completion) -> void { UNREFERENCED_PARAMETER(overridden_listener); - if (!config.internalListenerConfig().has_value()) { - config.listenSocketFactory().getListenSocket(); - } + config.listenSocketFactory().getListenSocket(); EXPECT_EQ(nullptr, add_listener_completion_); add_listener_completion_ = completion; })); diff --git a/test/server/BUILD b/test/server/BUILD index 1b8a0581563d1..c35ef66fa63b9 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -74,51 +74,19 @@ envoy_cc_test( name = "connection_handler_test", srcs = ["connection_handler_test.cc"], deps = [ - ":utility_lib", - "//source/common/api:os_sys_calls_lib", "//source/common/common:utility_lib", - "//source/common/config:metadata_lib", "//source/common/config:utility_lib", - "//source/common/init:manager_lib", - "//source/common/network:addr_family_aware_socket_option_lib", "//source/common/network:address_lib", "//source/common/network:connection_balancer_lib", - "//source/common/network:listen_socket_lib", - "//source/common/network:socket_option_lib", "//source/common/network:udp_packet_writer_handler_lib", - "//source/common/network:utility_lib", - "//source/common/protobuf", "//source/common/stats:stats_lib", - "//source/extensions/filters/listener/original_dst:config", - "//source/extensions/filters/listener/proxy_protocol:config", - "//source/extensions/filters/listener/tls_inspector:config", - "//source/extensions/filters/network/http_connection_manager:config", - "//source/extensions/filters/network/tcp_proxy:config", - "//source/extensions/request_id/uuid:config", - "//source/extensions/transport_sockets/raw_buffer:config", - "//source/extensions/transport_sockets/tls:config", - "//source/extensions/transport_sockets/tls:ssl_socket_lib", "//source/server:active_raw_udp_listener_config", "//source/server:connection_handler_lib", - "//source/server:listener_manager_lib", "//test/mocks/access_log:access_log_mocks", "//test/mocks/api:api_mocks", - "//test/mocks/init:init_mocks", "//test/mocks/network:network_mocks", - "//test/mocks/server:drain_manager_mocks", - "//test/mocks/server:guard_dog_mocks", - "//test/mocks/server:instance_mocks", - "//test/mocks/server:listener_component_factory_mocks", - "//test/mocks/server:worker_factory_mocks", - "//test/mocks/server:worker_mocks", - "//test/test_common:environment_lib", "//test/test_common:network_utility_lib", - "//test/test_common:registry_lib", - "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_runtime_lib", - "//test/test_common:test_time_lib", "//test/test_common:threadsafe_singleton_injector_lib", - "@envoy_api//envoy/admin/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", ], @@ -143,57 +111,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "active_internal_listener_test", - srcs = ["active_internal_listener_test.cc"], - deps = [ - ":utility_lib", - "//source/common/api:os_sys_calls_lib", - "//source/common/common:utility_lib", - "//source/common/config:metadata_lib", - "//source/common/config:utility_lib", - "//source/common/init:manager_lib", - "//source/common/network:addr_family_aware_socket_option_lib", - "//source/common/network:address_lib", - "//source/common/network:connection_balancer_lib", - "//source/common/network:listen_socket_lib", - "//source/common/network:socket_option_lib", - "//source/common/network:utility_lib", - "//source/common/protobuf", - "//source/common/stats:stats_lib", - "//source/extensions/filters/listener/original_dst:config", - "//source/extensions/filters/listener/proxy_protocol:config", - "//source/extensions/filters/listener/tls_inspector:config", - "//source/extensions/filters/network/http_connection_manager:config", - "//source/extensions/filters/network/tcp_proxy:config", - "//source/extensions/request_id/uuid:config", - "//source/extensions/transport_sockets/raw_buffer:config", - "//source/extensions/transport_sockets/tls:config", - "//source/extensions/transport_sockets/tls:ssl_socket_lib", - "//source/server:active_raw_udp_listener_config", - "//source/server:connection_handler_lib", - "//source/server:listener_manager_lib", - "//test/mocks/access_log:access_log_mocks", - "//test/mocks/api:api_mocks", - "//test/mocks/init:init_mocks", - "//test/mocks/network:network_mocks", - "//test/mocks/server:drain_manager_mocks", - "//test/mocks/server:guard_dog_mocks", - "//test/mocks/server:instance_mocks", - "//test/mocks/server:listener_component_factory_mocks", - "//test/mocks/server:worker_factory_mocks", - "//test/mocks/server:worker_mocks", - "//test/test_common:environment_lib", - "//test/test_common:network_utility_lib", - "//test/test_common:registry_lib", - "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_runtime_lib", - "//test/test_common:test_time_lib", - "//test/test_common:threadsafe_singleton_injector_lib", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "drain_manager_impl_test", srcs = ["drain_manager_impl_test.cc"], diff --git a/test/server/active_internal_listener_test.cc b/test/server/active_internal_listener_test.cc deleted file mode 100644 index 71d170dba3d91..0000000000000 --- a/test/server/active_internal_listener_test.cc +++ /dev/null @@ -1,187 +0,0 @@ -#include - -#include "envoy/config/core/v3/base.pb.h" -#include "envoy/network/exception.h" -#include "envoy/network/filter.h" -#include "envoy/network/listener.h" -#include "envoy/stats/scope.h" - -#include "source/common/common/utility.h" -#include "source/common/config/utility.h" -#include "source/common/network/address_impl.h" -#include "source/common/network/connection_balancer_impl.h" -#include "source/common/network/io_socket_handle_impl.h" -#include "source/common/network/raw_buffer_socket.h" -#include "source/common/network/utility.h" -#include "source/server/active_internal_listener.h" -#include "source/server/connection_handler_impl.h" - -#include "test/mocks/access_log/mocks.h" -#include "test/mocks/api/mocks.h" -#include "test/mocks/common.h" -#include "test/mocks/network/mocks.h" -#include "test/test_common/network_utility.h" -#include "test/test_common/threadsafe_singleton_injector.h" - -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -using testing::_; -using testing::Invoke; -using testing::NiceMock; -using testing::Return; -using testing::ReturnRef; - -namespace Envoy { -namespace Server { -namespace { - -class MockInternalListenerCallback : public Network::InternalListenerCallbacks { -public: - MOCK_METHOD(void, onAccept, (Network::ConnectionSocketPtr && socket), ()); - MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); -}; -class ActiveInternalListenerTest : public testing::Test, - protected Logger::Loggable { -public: - ActiveInternalListenerTest() { - EXPECT_CALL(listener_config_, listenerScope).Times(testing::AnyNumber()); - EXPECT_CALL(conn_handler_, statPrefix()).WillRepeatedly(ReturnRef(listener_stat_prefix_)); - listener_filter_matcher_ = std::make_shared>(); - } - void addListener() { - EXPECT_CALL(listener_config_, listenerFiltersTimeout()); - EXPECT_CALL(listener_config_, continueOnListenerFiltersTimeout()); - EXPECT_CALL(listener_config_, filterChainManager()).WillRepeatedly(ReturnRef(manager_)); - EXPECT_CALL(listener_config_, openConnections()).WillRepeatedly(ReturnRef(resource_limit_)); - auto mock_listener_will_be_moved = std::make_unique(); - generic_listener_ = mock_listener_will_be_moved.get(); - internal_listener_ = std::make_shared( - conn_handler_, dispatcher_, std::move(mock_listener_will_be_moved), listener_config_); - } - Network::Listener* addListenerWithRealNetworkListener() { - EXPECT_CALL(listener_config_, listenerFiltersTimeout()); - EXPECT_CALL(listener_config_, continueOnListenerFiltersTimeout()); - EXPECT_CALL(listener_config_, filterChainManager()).WillRepeatedly(ReturnRef(manager_)); - EXPECT_CALL(listener_config_, openConnections()).WillRepeatedly(ReturnRef(resource_limit_)); - - internal_listener_ = - std::make_shared(conn_handler_, dispatcher_, listener_config_); - return internal_listener_->listener(); - } - void expectFilterChainFactory() { - EXPECT_CALL(listener_config_, filterChainFactory()) - .WillRepeatedly(ReturnRef(filter_chain_factory_)); - } - std::string listener_stat_prefix_{"listener_stat_prefix"}; - NiceMock dispatcher_{"test"}; - BasicResourceLimitImpl resource_limit_; - Network::MockConnectionHandler conn_handler_; - Network::MockListener* generic_listener_; - Network::MockListenerConfig listener_config_; - NiceMock manager_; - NiceMock filter_chain_factory_; - std::shared_ptr filter_chain_; - std::shared_ptr> listener_filter_matcher_; - std::shared_ptr internal_listener_; -}; - -TEST_F(ActiveInternalListenerTest, BasicInternalListener) { - addListener(); - EXPECT_CALL(*generic_listener_, onDestroy()); -} - -TEST_F(ActiveInternalListenerTest, AcceptSocketAndCreateListenerFilter) { - addListener(); - expectFilterChainFactory(); - Network::MockListenerFilter* test_listener_filter = new Network::MockListenerFilter(); - // FIX-ME: replace by mock socket - Network::Address::InstanceConstSharedPtr original_dst_address( - new Network::Address::Ipv4Instance("127.0.0.2", 8080)); - - Network::MockConnectionSocket* accepted_socket = new NiceMock(); - - EXPECT_CALL(filter_chain_factory_, createListenerFilterChain(_)) - .WillRepeatedly(Invoke([&](Network::ListenerFilterManager& manager) -> bool { - // Insert the Mock filter. - manager.addAcceptFilter(listener_filter_matcher_, - Network::ListenerFilterPtr{test_listener_filter}); - return true; - })); - EXPECT_CALL(*test_listener_filter, onAccept(_)) - .WillOnce(Invoke([&](Network::ListenerFilterCallbacks& cb) -> Network::FilterStatus { - cb.socket().addressProvider().restoreLocalAddress(original_dst_address); - return Network::FilterStatus::Continue; - })); - EXPECT_CALL(*test_listener_filter, destroy_()); - EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(nullptr)); - internal_listener_->onAccept(Network::ConnectionSocketPtr{accepted_socket}); - EXPECT_CALL(*generic_listener_, onDestroy()); -} - -TEST_F(ActiveInternalListenerTest, AcceptSocketAndCreateNetworkFilter) { - - addListener(); - expectFilterChainFactory(); - - Network::MockListenerFilter* test_listener_filter = new Network::MockListenerFilter(); - // FIX-ME: replace by mock socket - Network::Address::InstanceConstSharedPtr original_dst_address( - new Network::Address::Ipv4Instance("127.0.0.2", 8080)); - - Network::MockConnectionSocket* accepted_socket = new NiceMock(); - - EXPECT_CALL(filter_chain_factory_, createListenerFilterChain(_)) - .WillRepeatedly(Invoke([&](Network::ListenerFilterManager& manager) -> bool { - // Insert the Mock filter. - manager.addAcceptFilter(listener_filter_matcher_, - Network::ListenerFilterPtr{test_listener_filter}); - return true; - })); - EXPECT_CALL(*test_listener_filter, onAccept(_)) - .WillOnce(Invoke([&](Network::ListenerFilterCallbacks& cb) -> Network::FilterStatus { - cb.socket().addressProvider().restoreLocalAddress(original_dst_address); - return Network::FilterStatus::Continue; - })); - EXPECT_CALL(*test_listener_filter, destroy_()); - auto filter_factory_callback = std::make_shared>(); - filter_chain_ = std::make_shared>(); - auto transport_socket_factory = Network::Test::createRawBufferSocketFactory(); - - EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(filter_chain_.get())); - EXPECT_CALL(*filter_chain_, transportSocketFactory) - .WillOnce(testing::ReturnRef(*transport_socket_factory)); - EXPECT_CALL(*filter_chain_, networkFilterFactories).WillOnce(ReturnRef(*filter_factory_callback)); - auto* connection = new NiceMock(); - EXPECT_CALL(dispatcher_, createServerConnection_()).WillOnce(Return(connection)); - EXPECT_CALL(conn_handler_, incNumConnections()); - EXPECT_CALL(filter_chain_factory_, createNetworkFilterChain(_, _)).WillOnce(Return(true)); - EXPECT_CALL(listener_config_, perConnectionBufferLimitBytes()); - internal_listener_->onAccept(Network::ConnectionSocketPtr{accepted_socket}); - EXPECT_CALL(conn_handler_, decNumConnections()); - connection->close(Network::ConnectionCloseType::NoFlush); - dispatcher_.clearDeferredDeleteList(); - EXPECT_CALL(*generic_listener_, onDestroy()); -} - -TEST_F(ActiveInternalListenerTest, StopListener) { - addListener(); - EXPECT_CALL(*generic_listener_, onDestroy()); - internal_listener_->shutdownListener(); -} - -TEST_F(ActiveInternalListenerTest, PausedListenerAcceptNewSocket) { - addListenerWithRealNetworkListener(); - internal_listener_->pauseListening(); - - expectFilterChainFactory(); - Network::MockConnectionSocket* accepted_socket = new NiceMock(); - - EXPECT_CALL(filter_chain_factory_, createListenerFilterChain(_)) - .WillRepeatedly(Invoke([&](Network::ListenerFilterManager&) -> bool { return true; })); - EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(nullptr)); - internal_listener_->onAccept(Network::ConnectionSocketPtr{accepted_socket}); -} -} // namespace -} // namespace Server -} // namespace Envoy diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 3bebfaaef4dd2..b97a3d140c614 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -1,52 +1,3 @@ -#include -#include -#include -#include -#include - -#include "envoy/admin/v3/config_dump.pb.h" -#include "envoy/config/core/v3/address.pb.h" -#include "envoy/config/core/v3/base.pb.h" -#include "envoy/config/core/v3/config_source.pb.h" -#include "envoy/config/listener/v3/listener.pb.h" -#include "envoy/network/listener.h" -#include "envoy/server/filter_config.h" -#include "envoy/server/listener_manager.h" -#include "envoy/stream_info/filter_state.h" - -#include "source/common/api/os_sys_calls_impl.h" -#include "source/common/common/macros.h" -#include "source/common/config/metadata.h" -#include "source/common/init/manager_impl.h" -#include "source/common/network/address_impl.h" -#include "source/common/network/io_socket_handle_impl.h" -#include "source/common/network/utility.h" -#include "source/common/protobuf/protobuf.h" -#include "source/server/configuration_impl.h" -#include "source/server/listener_manager_impl.h" - -#include "test/mocks/init/mocks.h" -#include "test/mocks/network/mocks.h" -#include "test/mocks/server/drain_manager.h" -#include "test/mocks/server/guard_dog.h" -#include "test/mocks/server/instance.h" -#include "test/mocks/server/listener_component_factory.h" -#include "test/mocks/server/worker.h" -#include "test/mocks/server/worker_factory.h" -#include "test/server/utility.h" -#include "test/test_common/environment.h" -#include "test/test_common/network_utility.h" -#include "test/test_common/registry.h" -#include "test/test_common/simulated_time_system.h" -#include "test/test_common/test_runtime.h" -#include "test/test_common/threadsafe_singleton_injector.h" -#include "test/test_common/utility.h" - -#include "absl/strings/escaping.h" -#include "absl/strings/match.h" - -// Above from listener_manager_impl_test.cc - #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/listener/v3/udp_listener_config.pb.h" #include "envoy/network/exception.h" @@ -155,8 +106,6 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable udp_listener_config_; - std::unique_ptr internal_listener_config_; Network::ConnectionBalancerSharedPtr connection_balancer_; BasicResourceLimitImpl open_connections_; const std::vector access_logs_; @@ -313,22 +254,6 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable> overridden_filter_chain_manager = - nullptr) { - listeners_.emplace_back(std::make_unique( - *this, tag, /*bind_to_port*/ false, /*hand_off_restored_destination_connections*/ false, - name, Network::Socket::Type::Stream, listener_filters_timeout, - continue_on_listener_filters_timeout, socket_factory_, access_log_, - overridden_filter_chain_manager, ENVOY_TCP_BACKLOG_SIZE, nullptr)); - listeners_.back()->internal_listener_config_ = - std::make_unique(); - return listeners_.back().get(); - } - void validateOriginalDst(Network::TcpListenerCallbacks** listener_callbacks, TestListener* test_listener, Network::MockListener* listener) { Network::Address::InstanceConstSharedPtr normal_address( @@ -373,7 +298,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable dispatcher_{"test"}; std::list listeners_; - std::unique_ptr handler_; + Network::ConnectionHandlerPtr handler_; NiceMock manager_; NiceMock factory_; const std::shared_ptr filter_chain_; @@ -1442,67 +1367,6 @@ TEST_F(ConnectionHandlerTest, TcpBacklogCustom) { handler_->addListener(absl::nullopt, *test_listener); } -TEST_F(ConnectionHandlerTest, DisableInternalListener) { - InSequence s; - Network::Address::InstanceConstSharedPtr local_address{ - new Network::Address::EnvoyInternalInstance("server_internal_address")}; - - TestListener* internal_listener = - addInternalListener(1, "test_internal_listener", std::chrono::milliseconds(), false, nullptr); - EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address)); - handler_->addListener(absl::nullopt, *internal_listener); - auto internal_listener_cb = handler_->findByAddress(local_address); - ASSERT(internal_listener_cb.has_value()); - - handler_->disableListeners(); - auto internal_listener_cb_disabled = handler_->findByAddress(local_address); - ASSERT(internal_listener_cb_disabled.has_value()); - ASSERT_EQ(&internal_listener_cb_disabled.value().get(), &internal_listener_cb.value().get()); - - handler_->enableListeners(); - auto internal_listener_cb_enabled = handler_->findByAddress(local_address); - ASSERT(internal_listener_cb_enabled.has_value()); - ASSERT_EQ(&internal_listener_cb_enabled.value().get(), &internal_listener_cb.value().get()); -} - -TEST_F(ConnectionHandlerTest, InternalListenerInplaceUpdate) { - InSequence s; - uint64_t old_listener_tag = 1; - uint64_t new_listener_tag = 2; - Network::Address::InstanceConstSharedPtr local_address{ - new Network::Address::EnvoyInternalInstance("server_internal_address")}; - - TestListener* internal_listener = addInternalListener( - old_listener_tag, "test_internal_listener", std::chrono::milliseconds(), false, nullptr); - EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address)); - handler_->addListener(absl::nullopt, *internal_listener); - - ASSERT_NE(internal_listener, nullptr); - - auto overridden_filter_chain_manager = - std::make_shared>(); - TestListener* new_test_listener = - addInternalListener(new_listener_tag, "test_internal_listener", std::chrono::milliseconds(), - false, overridden_filter_chain_manager); - - handler_->addListener(old_listener_tag, *new_test_listener); - - Network::MockConnectionSocket* connection = new NiceMock(); - - auto internal_listener_cb = handler_->findByAddress(local_address); - - EXPECT_CALL(manager_, findFilterChain(_)).Times(0); - EXPECT_CALL(*overridden_filter_chain_manager, findFilterChain(_)).WillOnce(Return(nullptr)); - EXPECT_CALL(*access_log_, log(_, _, _, _)); - internal_listener_cb.value().get().onAccept(Network::ConnectionSocketPtr{connection}); - EXPECT_EQ(0UL, handler_->numConnections()); - - testing::MockFunction completion; - handler_->removeFilterChains(old_listener_tag, {}, completion.AsStdFunction()); - EXPECT_CALL(completion, Call()); - dispatcher_.clearDeferredDeleteList(); -} - } // namespace } // namespace Server } // namespace Envoy diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 0ca841cd36883..40ee2fecc83ff 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -580,10 +580,6 @@ bind_to_port: false } TEST_F(ListenerManagerImplTest, UnsupportedInternalListener) { - auto scoped_runtime_guard = std::make_unique(); - - Runtime::LoaderSingleton::getExisting()->mergeValues( - {{"envoy.reloadable_features.internal_address", "false"}}); const std::string yaml = R"EOF( address: envoy_internal_address: @@ -4728,313 +4724,6 @@ name: test_api_listener_2 EXPECT_EQ("test_api_listener", manager_->apiListener()->get().name()); } -TEST_F(ListenerManagerImplWithRealFiltersTest, AddOrUpdateInternalListener) { - auto scoped_runtime_guard = std::make_unique(); - Runtime::LoaderSingleton::getExisting()->mergeValues( - {{"envoy.reloadable_features.internal_address", "true"}}); - time_system_.setSystemTime(std::chrono::milliseconds(1001001001001)); - - InSequence s; - - auto* lds_api = new MockLdsApi(); - EXPECT_CALL(listener_factory_, createLdsApi_(_, _)).WillOnce(Return(lds_api)); - envoy::config::core::v3::ConfigSource lds_config; - manager_->createLdsApi(lds_config, nullptr); - - EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("")); - checkConfigDump(R"EOF( -static_listeners: -)EOF"); - - // Add foo listener. - const std::string listener_foo_yaml = R"EOF( -name: test_internal_listener -address: - envoy_internal_address: - server_listener_name: test_internal_listener_name -internal_listener: {} -filter_chains: {} - )EOF"; - - ListenerHandle* listener_foo = expectListenerCreate(false, true); - // EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); - EXPECT_TRUE( - manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml), "version1", true)); - checkStats(__LINE__, 1, 0, 0, 0, 1, 0, 0); - EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("version1")); - checkConfigDump(R"EOF( -version_info: version1 -static_listeners: -dynamic_listeners: - - name: test_internal_listener - warming_state: - version_info: version1 - listener: - "@type": type.googleapis.com/envoy.config.listener.v3.Listener - name: test_internal_listener - address: - envoy_internal_address: - server_listener_name: test_internal_listener_name - filter_chains: {} - internal_listener: {} - last_updated: - seconds: 1001001001 - nanos: 1000000 -)EOF"); - - // Update duplicate should be a NOP. - EXPECT_FALSE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml), "", true)); - checkStats(__LINE__, 1, 0, 0, 0, 1, 0, 0); - - // Update foo listener. Should share socket. - const std::string listener_foo_update1_yaml = R"EOF( -name: test_internal_listener -address: - envoy_internal_address: - server_listener_name: test_internal_listener_name -filter_chains: {} -internal_listener: {} -per_connection_buffer_limit_bytes: 10 - )EOF"; - - time_system_.setSystemTime(std::chrono::milliseconds(2002002002002)); - - ListenerHandle* listener_foo_update1 = expectListenerCreate(false, true); - EXPECT_CALL(*listener_foo, onDestroy()); - EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_update1_yaml), - "version2", true)); - checkStats(__LINE__, 1, 1, 0, 0, 1, 0, 0); - EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("version2")); - checkConfigDump(R"EOF( - version_info: version2 - static_listeners: - dynamic_listeners: - - name: test_internal_listener - warming_state: - version_info: version2 - listener: - "@type": type.googleapis.com/envoy.config.listener.v3.Listener - name: test_internal_listener - address: - envoy_internal_address: - server_listener_name: test_internal_listener_name - filter_chains: {} - internal_listener: {} - per_connection_buffer_limit_bytes: 10 - last_updated: - seconds: 2002002002 - nanos: 2000000 - )EOF"); - - // Validate that workers_started stat is zero before calling startWorkers. - EXPECT_EQ(0, server_.stats_store_ - .gauge("listener_manager.workers_started", Stats::Gauge::ImportMode::NeverImport) - .value()); - - // Start workers. - EXPECT_CALL(*worker_, addListener(_, _, _)); - EXPECT_CALL(*worker_, start(_)); - manager_->startWorkers(guard_dog_, callback_.AsStdFunction()); - // Validate that workers_started stat is still zero before workers set the status via - // completion callback. - EXPECT_EQ(0, server_.stats_store_ - .gauge("listener_manager.workers_started", Stats::Gauge::ImportMode::NeverImport) - .value()); - worker_->callAddCompletion(true); - - // Validate that workers_started stat is set to 1 after workers have responded with initialization - // status. - EXPECT_EQ(1, server_.stats_store_ - .gauge("listener_manager.workers_started", Stats::Gauge::ImportMode::NeverImport) - .value()); - - // Update duplicate should be a NOP. - EXPECT_FALSE( - manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_update1_yaml), "", true)); - checkStats(__LINE__, 1, 1, 0, 0, 1, 0, 0); - - time_system_.setSystemTime(std::chrono::milliseconds(3003003003003)); - - // Update foo. Should go into warming, have an immediate warming callback, and start immediate - // removal. - ListenerHandle* listener_foo_update2 = expectListenerCreate(false, true); - EXPECT_CALL(*worker_, addListener(_, _, _)); - EXPECT_CALL(*worker_, stopListener(_, _)); - EXPECT_CALL(*listener_foo_update1->drain_manager_, startDrainSequence(_)); - EXPECT_TRUE( - manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_foo_yaml), "version3", true)); - worker_->callAddCompletion(true); - checkStats(__LINE__, 1, 2, 0, 0, 1, 1, 0); - EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("version3")); - checkConfigDump(R"EOF( - version_info: version3 - static_listeners: - dynamic_listeners: - - name: test_internal_listener - active_state: - version_info: version3 - listener: - "@type": type.googleapis.com/envoy.config.listener.v3.Listener - name: test_internal_listener - address: - envoy_internal_address: - server_listener_name: test_internal_listener_name - filter_chains: {} - internal_listener: {} - last_updated: - seconds: 3003003003 - nanos: 3000000 - draining_state: - version_info: version2 - listener: - "@type": type.googleapis.com/envoy.config.listener.v3.Listener - name: test_internal_listener - address: - envoy_internal_address: - server_listener_name: test_internal_listener_name - filter_chains: {} - internal_listener: {} - per_connection_buffer_limit_bytes: 10 - last_updated: - seconds: 2002002002 - nanos: 2000000 - )EOF"); - - EXPECT_CALL(*worker_, removeListener(_, _)); - listener_foo_update1->drain_manager_->drain_sequence_completion_(); - checkStats(__LINE__, 1, 2, 0, 0, 1, 1, 0); - EXPECT_CALL(*listener_foo_update1, onDestroy()); - worker_->callRemovalCompletion(); - checkStats(__LINE__, 1, 2, 0, 0, 1, 0, 0); - - time_system_.setSystemTime(std::chrono::milliseconds(4004004004004)); - - // Add bar listener. - const std::string listener_bar_yaml = R"EOF( - name: test_internal_listener_bar - address: - envoy_internal_address: - server_listener_name: test_internal_listener_bar - filter_chains: {} - internal_listener: {} - )EOF"; - - ListenerHandle* listener_bar = expectListenerCreate(false, true); - // EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); - EXPECT_CALL(*worker_, addListener(_, _, _)); - EXPECT_TRUE( - manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_bar_yaml), "version4", true)); - EXPECT_EQ(2UL, manager_->listeners().size()); - worker_->callAddCompletion(true); - checkStats(__LINE__, 2, 2, 0, 0, 2, 0, 0); - - time_system_.setSystemTime(std::chrono::milliseconds(5005005005005)); - - // Add baz listener, this time requiring initializing. - const std::string listener_baz_yaml = R"EOF( - name: test_internal_listener_baz - address: - envoy_internal_address: - server_listener_name: test_internal_listener_baz - filter_chains: {} - internal_listener: {} - )EOF"; - - ListenerHandle* listener_baz = expectListenerCreate(true, true); - // EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, {true})); - EXPECT_CALL(listener_baz->target_, initialize()); - EXPECT_TRUE( - manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_baz_yaml), "version5", true)); - EXPECT_EQ(2UL, manager_->listeners().size()); - checkStats(__LINE__, 3, 2, 0, 1, 2, 0, 0); - EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("version5")); - checkConfigDump(R"EOF( - version_info: version5 - dynamic_listeners: - - name: test_internal_listener - active_state: - version_info: version3 - listener: - "@type": type.googleapis.com/envoy.config.listener.v3.Listener - name: test_internal_listener - address: - envoy_internal_address: - server_listener_name: test_internal_listener_name - filter_chains: {} - internal_listener: {} - last_updated: - seconds: 3003003003 - nanos: 3000000 - - name: test_internal_listener_bar - active_state: - version_info: version4 - listener: - "@type": type.googleapis.com/envoy.config.listener.v3.Listener - name: test_internal_listener_bar - address: - envoy_internal_address: - server_listener_name: test_internal_listener_bar - filter_chains: {} - internal_listener: {} - last_updated: - seconds: 4004004004 - nanos: 4000000 - - name: test_internal_listener_baz - warming_state: - version_info: version5 - listener: - "@type": type.googleapis.com/envoy.config.listener.v3.Listener - name: test_internal_listener_baz - address: - envoy_internal_address: - server_listener_name: test_internal_listener_baz - filter_chains: {} - internal_listener: {} - last_updated: - seconds: 5005005005 - nanos: 5000000 - )EOF"); - - // Update a duplicate baz that is currently warming. - EXPECT_FALSE(manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_baz_yaml), "", true)); - checkStats(__LINE__, 3, 2, 0, 1, 2, 0, 0); - - // Update baz while it is warming. - const std::string listener_baz_update1_yaml = R"EOF( - name: test_internal_listener_baz - address: - envoy_internal_address: - server_listener_name: test_internal_listener_baz - internal_listener: {} - filter_chains: - - filters: - - name: fake - typed_config: {} - )EOF"; - - ListenerHandle* listener_baz_update1 = expectListenerCreate(true, true); - EXPECT_CALL(*listener_baz, onDestroy()).WillOnce(Invoke([listener_baz]() -> void { - // Call the initialize callback during destruction like RDS will. - listener_baz->target_.ready(); - })); - EXPECT_CALL(listener_baz_update1->target_, initialize()); - EXPECT_TRUE( - manager_->addOrUpdateListener(parseListenerFromV3Yaml(listener_baz_update1_yaml), "", true)); - EXPECT_EQ(2UL, manager_->listeners().size()); - checkStats(__LINE__, 3, 3, 0, 1, 2, 0, 0); - - // Finish initialization for baz which should make it active. - EXPECT_CALL(*worker_, addListener(_, _, _)); - listener_baz_update1->target_.ready(); - EXPECT_EQ(3UL, manager_->listeners().size()); - worker_->callAddCompletion(true); - checkStats(__LINE__, 3, 3, 0, 0, 3, 0, 0); - - EXPECT_CALL(*listener_foo_update2, onDestroy()); - EXPECT_CALL(*listener_bar, onDestroy()); - EXPECT_CALL(*listener_baz_update1, onDestroy()); -} - TEST_F(ListenerManagerImplTest, StopInplaceWarmingListener) { InSequence s; From 9e5b2b4dceef9f21cb9e689678b48f4e66921c46 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 11 Jun 2021 23:04:02 +0000 Subject: [PATCH 08/23] pragma once Signed-off-by: Yuchen Dai --- source/server/active_stream_socket.h | 1 - 1 file changed, 1 deletion(-) diff --git a/source/server/active_stream_socket.h b/source/server/active_stream_socket.h index 61810c2444101..c85fc7825126a 100644 --- a/source/server/active_stream_socket.h +++ b/source/server/active_stream_socket.h @@ -1,6 +1,5 @@ #pragma once -#pragma once #include #include #include From e802a6ffcefb6f4e6cf0bdbccef9c70b1d5e147b Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 11 Jun 2021 23:07:34 +0000 Subject: [PATCH 09/23] test bootstrap Signed-off-by: Yuchen Dai --- source/server/active_stream_socket.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/server/active_stream_socket.h b/source/server/active_stream_socket.h index c85fc7825126a..a24ed03405236 100644 --- a/source/server/active_stream_socket.h +++ b/source/server/active_stream_socket.h @@ -30,7 +30,7 @@ namespace Server { class ActiveStreamListenerBase; /** - * Wrapper for an active accepted socket owned by the active listener. + * Wrapper for an active accepted socket owned by the active tcp listener. */ struct ActiveStreamSocket : public Network::ListenerFilterManager, public Network::ListenerFilterCallbacks, From c8e4e4338f30a636b94c491a4c091b9a882dc6a0 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 15 Jun 2021 17:40:17 +0000 Subject: [PATCH 10/23] comment other than incNumConnections Signed-off-by: Yuchen Dai --- source/server/active_stream_socket.cc | 26 +++++++++-------- source/server/active_stream_socket.h | 41 ++++++++++++++------------- source/server/active_tcp_listener.cc | 4 +-- source/server/active_tcp_listener.h | 1 - 4 files changed, 38 insertions(+), 34 deletions(-) diff --git a/source/server/active_stream_socket.cc b/source/server/active_stream_socket.cc index 88dc7239f5288..a1d6e8f1aedba 100644 --- a/source/server/active_stream_socket.cc +++ b/source/server/active_stream_socket.cc @@ -8,9 +8,9 @@ namespace Envoy { namespace Server { -ActiveStreamSocket::ActiveStreamSocket(ActiveStreamListenerBase& listener, - Network::ConnectionSocketPtr&& socket, - bool hand_off_restored_destination_connections) +ActiveTcpSocket::ActiveTcpSocket(ActiveStreamListenerBase& listener, + Network::ConnectionSocketPtr&& socket, + bool hand_off_restored_destination_connections) : listener_(listener), socket_(std::move(socket)), hand_off_restored_destination_connections_(hand_off_restored_destination_connections), iter_(accept_filters_.end()), @@ -19,7 +19,8 @@ ActiveStreamSocket::ActiveStreamSocket(ActiveStreamListenerBase& listener, StreamInfo::FilterState::LifeSpan::Connection)) { listener_.stats_.downstream_pre_cx_active_.inc(); } -ActiveStreamSocket::~ActiveStreamSocket() { + +ActiveTcpSocket::~ActiveTcpSocket() { accept_filters_.clear(); listener_.stats_.downstream_pre_cx_active_.dec(); @@ -34,7 +35,8 @@ ActiveStreamSocket::~ActiveStreamSocket() { listener_.decNumConnections(); } } -Event::Dispatcher& ActiveStreamSocket::dispatcher() { return listener_.dispatcher(); } + +Event::Dispatcher& ActiveTcpSocket::dispatcher() { return listener_.dispatcher(); } ActiveStreamListenerBase::ActiveStreamListenerBase(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, @@ -52,7 +54,7 @@ void ActiveStreamListenerBase::emitLogs(Network::ListenerConfig& config, } } -void ActiveStreamSocket::onTimeout() { +void ActiveTcpSocket::onTimeout() { listener_.stats_.downstream_pre_cx_timeout_.inc(); ASSERT(inserted()); ENVOY_LOG(debug, "listener filter times out after {} ms", @@ -65,14 +67,14 @@ void ActiveStreamSocket::onTimeout() { unlink(); } -void ActiveStreamSocket::startTimer() { +void ActiveTcpSocket::startTimer() { if (listener_.listener_filters_timeout_.count() > 0) { timer_ = listener_.dispatcher().createTimer([this]() -> void { onTimeout(); }); timer_->enableTimer(listener_.listener_filters_timeout_); } } -void ActiveStreamSocket::unlink() { +void ActiveTcpSocket::unlink() { auto removed = removeFromList(listener_.sockets_); if (removed->timer_ != nullptr) { removed->timer_->disableTimer(); @@ -84,7 +86,7 @@ void ActiveStreamSocket::unlink() { listener_.dispatcher().deferredDelete(std::move(removed)); } -void ActiveStreamSocket::continueFilterChain(bool success) { +void ActiveTcpSocket::continueFilterChain(bool success) { if (success) { bool no_error = true; if (iter_ == accept_filters_.end()) { @@ -123,12 +125,12 @@ void ActiveStreamSocket::continueFilterChain(bool success) { } } -void ActiveStreamSocket::setDynamicMetadata(const std::string& name, - const ProtobufWkt::Struct& value) { +void ActiveTcpSocket::setDynamicMetadata(const std::string& name, + const ProtobufWkt::Struct& value) { stream_info_->setDynamicMetadata(name, value); } -void ActiveStreamSocket::newConnection() { +void ActiveTcpSocket::newConnection() { connected_ = true; // Check if the socket may need to be redirected to another listener. diff --git a/source/server/active_stream_socket.h b/source/server/active_stream_socket.h index a24ed03405236..e0731a86c65f5 100644 --- a/source/server/active_stream_socket.h +++ b/source/server/active_stream_socket.h @@ -32,14 +32,14 @@ class ActiveStreamListenerBase; /** * Wrapper for an active accepted socket owned by the active tcp listener. */ -struct ActiveStreamSocket : public Network::ListenerFilterManager, - public Network::ListenerFilterCallbacks, - LinkedObject, - public Event::DeferredDeletable, - Logger::Loggable { - ActiveStreamSocket(ActiveStreamListenerBase& listener, Network::ConnectionSocketPtr&& socket, - bool hand_off_restored_destination_connections); - ~ActiveStreamSocket() override; +struct ActiveTcpSocket : public Network::ListenerFilterManager, + public Network::ListenerFilterCallbacks, + LinkedObject, + public Event::DeferredDeletable, + Logger::Loggable { + ActiveTcpSocket(ActiveStreamListenerBase& listener, Network::ConnectionSocketPtr&& socket, + bool hand_off_restored_destination_connections); + ~ActiveTcpSocket() override; void onTimeout(); void startTimer(); @@ -106,8 +106,6 @@ struct ActiveStreamSocket : public Network::ListenerFilterManager, bool connected_{false}; }; -using ActiveInternalSocket = ActiveStreamSocket; - class ActiveStreamListenerBase : public ActiveListenerImplBase { public: ActiveStreamListenerBase(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, @@ -116,7 +114,6 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase { virtual void incNumConnections() PURE; virtual void decNumConnections() PURE; - virtual Event::Dispatcher& dispatcher() PURE; /** * Create a new connection from a socket accepted by the listener. @@ -127,7 +124,7 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase { virtual Network::BalancedConnectionHandlerOptRef getBalancedHandlerByAddress(const Network::Address::Instance& address) PURE; - void onSocketAccepted(std::unique_ptr active_socket) { + void onSocketAccepted(std::unique_ptr active_socket) { // Create and run the filters config_->filterChainFactory().createListenerFilterChain(*active_socket); active_socket->continueFilterChain(true); @@ -141,28 +138,35 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase { // If active_socket is about to be destructed, emit logs if a connection is not created. if (!active_socket->connected_) { emitLogs(*config_, *active_socket->stream_info_); + } else { + // If the active_socket is not connected, this socket is not promoted to active connection. + // Thus the stream_info_ is owned by this active socket. + ENVOY_BUG(active_socket->stream_info_ != nullptr, + "the unconnected active socket must have stream info."); } } } + Event::Dispatcher& dispatcher() { return dispatcher_; } + Network::ConnectionHandler& parent_; Event::Dispatcher& dispatcher_; Network::ListenerPtr listener_; const std::chrono::milliseconds listener_filters_timeout_; const bool continue_on_listener_filters_timeout_; - std::list> sockets_; + std::list> sockets_; }; -template +template class TypedActiveStreamListenerBase : public ActiveStreamListenerBase { public: + using ActiveConnectionCollectionType = typename ActiveConnectionType::CollectionType; TypedActiveStreamListenerBase(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, Network::ListenerPtr&& listener, Network::ListenerConfig& config) : ActiveStreamListenerBase(parent, dispatcher, std::move(listener), config) {} - using ActiveGenericConnectionPtr = std::unique_ptr; - using ActiveGenericConnectionsPtr = std::unique_ptr; + using ActiveConnectionPtr = std::unique_ptr; + using ActiveConnectionsPtr = std::unique_ptr; /** * Schedule to remove and destroy the active connections which are not tracked by listener @@ -213,8 +217,7 @@ class TypedActiveStreamListenerBase : public ActiveStreamListenerBase { dispatcher().clearDeferredDeleteList(); } - absl::node_hash_map - connections_by_context_; + absl::node_hash_map connections_by_context_; bool is_deleting_{false}; }; diff --git a/source/server/active_tcp_listener.cc b/source/server/active_tcp_listener.cc index 895200e818fd2..f5d7517fbd646 100644 --- a/source/server/active_tcp_listener.cc +++ b/source/server/active_tcp_listener.cc @@ -112,8 +112,8 @@ void ActiveTcpListener::onAcceptWorker(Network::ConnectionSocketPtr&& socket, } } - auto active_socket = std::make_unique( - *this, std::move(socket), hand_off_restored_destination_connections); + auto active_socket = std::make_unique(*this, std::move(socket), + hand_off_restored_destination_connections); onSocketAccepted(std::move(active_socket)); } diff --git a/source/server/active_tcp_listener.h b/source/server/active_tcp_listener.h index fa0d2d611960a..a3e74dd96aebf 100644 --- a/source/server/active_tcp_listener.h +++ b/source/server/active_tcp_listener.h @@ -95,7 +95,6 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, // ActiveListenerImplBase Network::Listener* listener() override { return listener_.get(); } - Event::Dispatcher& dispatcher() override { return dispatcher_; } Network::BalancedConnectionHandlerOptRef getBalancedHandlerByAddress(const Network::Address::Instance& address) override; From aac23c19a4a6082cf52de0c07f8b80eaef2baa88 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 15 Jun 2021 22:29:46 +0000 Subject: [PATCH 11/23] rename Signed-off-by: Yuchen Dai --- source/server/BUILD | 4 ++-- source/server/active_tcp_listener.h | 2 +- ..._stream_socket.cc => active_tcp_socket.cc} | 2 +- ...ve_stream_socket.h => active_tcp_socket.h} | 20 ++++++++++--------- 4 files changed, 15 insertions(+), 13 deletions(-) rename source/server/{active_stream_socket.cc => active_tcp_socket.cc} (99%) rename source/server/{active_stream_socket.h => active_tcp_socket.h} (93%) diff --git a/source/server/BUILD b/source/server/BUILD index 28947666d5c61..4702170bf1480 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -160,9 +160,9 @@ envoy_cc_library( envoy_cc_library( name = "active_stream_socket", - srcs = ["active_stream_socket.cc"], + srcs = ["active_tcp_socket.cc"], hdrs = [ - "active_stream_socket.h", + "active_tcp_socket.h", ], deps = [ ":active_listener_base", diff --git a/source/server/active_tcp_listener.h b/source/server/active_tcp_listener.h index a3e74dd96aebf..05f860a8a3091 100644 --- a/source/server/active_tcp_listener.h +++ b/source/server/active_tcp_listener.h @@ -7,7 +7,7 @@ #include "source/common/common/linked_object.h" #include "source/common/stream_info/stream_info_impl.h" #include "source/server/active_listener_base.h" -#include "source/server/active_stream_socket.h" +#include "source/server/active_tcp_socket.h" namespace Envoy { namespace Server { diff --git a/source/server/active_stream_socket.cc b/source/server/active_tcp_socket.cc similarity index 99% rename from source/server/active_stream_socket.cc rename to source/server/active_tcp_socket.cc index a1d6e8f1aedba..89761ce347689 100644 --- a/source/server/active_stream_socket.cc +++ b/source/server/active_tcp_socket.cc @@ -1,4 +1,4 @@ -#include "source/server/active_stream_socket.h" +#include "source/server/active_tcp_socket.h" #include "envoy/network/filter.h" #include "envoy/stats/scope.h" diff --git a/source/server/active_stream_socket.h b/source/server/active_tcp_socket.h similarity index 93% rename from source/server/active_stream_socket.h rename to source/server/active_tcp_socket.h index e0731a86c65f5..034cf5c95a942 100644 --- a/source/server/active_stream_socket.h +++ b/source/server/active_tcp_socket.h @@ -115,6 +115,8 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase { virtual void incNumConnections() PURE; virtual void decNumConnections() PURE; + Event::Dispatcher& dispatcher() { return dispatcher_; } + /** * Create a new connection from a socket accepted by the listener. */ @@ -135,20 +137,20 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase { active_socket->startTimer(); LinkedList::moveIntoListBack(std::move(active_socket), sockets_); } else { - // If active_socket is about to be destructed, emit logs if a connection is not created. if (!active_socket->connected_) { - emitLogs(*config_, *active_socket->stream_info_); - } else { - // If the active_socket is not connected, this socket is not promoted to active connection. - // Thus the stream_info_ is owned by this active socket. - ENVOY_BUG(active_socket->stream_info_ != nullptr, - "the unconnected active socket must have stream info."); + // If active_socket is about to be destructed, emit logs if a connection is not created. + if (active_socket->stream_info_ != nullptr) { + emitLogs(*config_, *active_socket->stream_info_); + } else { + // If the active_socket is not connected, this socket is not promoted to active + // connection. Thus the stream_info_ is owned by this active socket. + ENVOY_BUG(active_socket->stream_info_ != nullptr, + "the unconnected active socket must have stream info."); + } } } } - Event::Dispatcher& dispatcher() { return dispatcher_; } - Network::ConnectionHandler& parent_; Event::Dispatcher& dispatcher_; Network::ListenerPtr listener_; From 573cdfeca1448022ae9a894323d3769d3e5fbf77 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 17 Jun 2021 07:59:01 +0000 Subject: [PATCH 12/23] put cleanup, deferdelete in base Signed-off-by: Yuchen Dai --- source/server/BUILD | 4 +- source/server/active_tcp_listener.cc | 21 ++++++- source/server/active_tcp_listener.h | 2 +- source/server/active_tcp_socket.h | 86 ++++++++++++---------------- 4 files changed, 59 insertions(+), 54 deletions(-) diff --git a/source/server/BUILD b/source/server/BUILD index 4702170bf1480..3f6d6d32805f8 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -116,7 +116,7 @@ envoy_cc_library( "active_tcp_listener.h", ], deps = [ - ":active_stream_socket", + ":active_tcp_socket", "//envoy/common:time_interface", "//envoy/event:deferred_deletable", "//envoy/event:dispatcher_interface", @@ -159,7 +159,7 @@ envoy_cc_library( ) envoy_cc_library( - name = "active_stream_socket", + name = "active_tcp_socket", srcs = ["active_tcp_socket.cc"], hdrs = [ "active_tcp_socket.h", diff --git a/source/server/active_tcp_listener.cc b/source/server/active_tcp_listener.cc index f5d7517fbd646..048820fbd4f9e 100644 --- a/source/server/active_tcp_listener.cc +++ b/source/server/active_tcp_listener.cc @@ -39,7 +39,23 @@ ActiveTcpListener::ActiveTcpListener(Network::TcpConnectionHandler& parent, ActiveTcpListener::~ActiveTcpListener() { config_->connectionBalancer().unregisterHandler(*this); - cleanupConnections(); + is_deleting_ = true; + + // Purge sockets that have not progressed to connections. This should only happen when + // a listener filter stops iteration and never resumes. + while (!sockets_.empty()) { + auto removed = sockets_.front()->removeFromList(sockets_); + dispatcher().deferredDelete(std::move(removed)); + } + + for (auto& chain_and_connections : connections_by_context_) { + ASSERT(chain_and_connections.second != nullptr); + auto& connections = chain_and_connections.second->connections_; + while (!connections.empty()) { + connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); + } + } + dispatcher().clearDeferredDeleteList(); // By the time a listener is destroyed, in the common case, there should be no connections. // However, this is not always true if there is an in flight rebalanced connection that is @@ -189,7 +205,7 @@ void ActiveTcpListener::newConnection(Network::ConnectionSocketPtr&& socket, ActiveConnections& ActiveTcpListener::getOrCreateActiveConnections(const Network::FilterChain& filter_chain) { - ActiveConnectionsPtr& connections = connections_by_context_[&filter_chain]; + ActiveConnectionCollectionPtr& connections = connections_by_context_[&filter_chain]; if (connections == nullptr) { connections = std::make_unique(*this, filter_chain); } @@ -263,7 +279,6 @@ ActiveTcpConnection::~ActiveTcpConnection() { listener.parent_.decNumConnections(); } -// Network::ConnectionCallbacks void ActiveTcpConnection::onEvent(Network::ConnectionEvent event) { ENVOY_LOG(trace, "[C{}] connection on event {}", connection_->id(), static_cast(event)); // Any event leads to destruction of the connection. diff --git a/source/server/active_tcp_listener.h b/source/server/active_tcp_listener.h index 05f860a8a3091..a62855b6115de 100644 --- a/source/server/active_tcp_listener.h +++ b/source/server/active_tcp_listener.h @@ -15,7 +15,7 @@ namespace Server { struct ActiveTcpConnection; using ActiveTcpConnectionPtr = std::unique_ptr; class ActiveConnections; -using ActiveConnectionsPtr = std::unique_ptr; +using ActiveConnectionCollectionPtr = std::unique_ptr; class ActiveTcpListener; diff --git a/source/server/active_tcp_socket.h b/source/server/active_tcp_socket.h index 034cf5c95a942..9424009a1ca3a 100644 --- a/source/server/active_tcp_socket.h +++ b/source/server/active_tcp_socket.h @@ -112,17 +112,37 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase { Network::ListenerPtr&& listener, Network::ListenerConfig& config); static void emitLogs(Network::ListenerConfig& config, StreamInfo::StreamInfo& stream_info); + Event::Dispatcher& dispatcher() { return dispatcher_; } + + /** + * Schedule to remove and destroy the active connections which are not tracked by listener + * config. Caution: The connection are not destroyed yet when function returns. + */ + void + deferredRemoveFilterChains(const std::list& draining_filter_chains) { + // Need to recover the original deleting state. + const bool was_deleting = is_deleting_; + is_deleting_ = true; + for (const auto* filter_chain : draining_filter_chains) { + deferRemoveFilterChain(filter_chain); + } + is_deleting_ = was_deleting; + } + virtual void incNumConnections() PURE; virtual void decNumConnections() PURE; - Event::Dispatcher& dispatcher() { return dispatcher_; } - /** * Create a new connection from a socket accepted by the listener. */ virtual void newConnection(Network::ConnectionSocketPtr&& socket, std::unique_ptr stream_info) PURE; + /** + * Schedule to remove and destroy the active connections owned by the filter chain. + */ + virtual void deferRemoveFilterChain(const Network::FilterChain* filter_chain) PURE; + virtual Network::BalancedConnectionHandlerOptRef getBalancedHandlerByAddress(const Network::Address::Instance& address) PURE; @@ -157,6 +177,7 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase { const std::chrono::milliseconds listener_filters_timeout_; const bool continue_on_listener_filters_timeout_; std::list> sockets_; + bool is_deleting_{false}; }; template @@ -168,59 +189,28 @@ class TypedActiveStreamListenerBase : public ActiveStreamListenerBase { Network::ListenerPtr&& listener, Network::ListenerConfig& config) : ActiveStreamListenerBase(parent, dispatcher, std::move(listener), config) {} using ActiveConnectionPtr = std::unique_ptr; - using ActiveConnectionsPtr = std::unique_ptr; + using ActiveConnectionCollectionPtr = std::unique_ptr; - /** - * Schedule to remove and destroy the active connections which are not tracked by listener - * config. Caution: The connection are not destroyed yet when function returns. - */ - void - deferredRemoveFilterChains(const std::list& draining_filter_chains) { - // Need to recover the original deleting state. - const bool was_deleting = is_deleting_; - is_deleting_ = true; - for (const auto* filter_chain : draining_filter_chains) { - auto iter = connections_by_context_.find(filter_chain); - if (iter == connections_by_context_.end()) { - // It is possible when listener is stopping. - } else { - auto& connections = iter->second->connections_; - while (!connections.empty()) { - connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); - } - // Since is_deleting_ is on, we need to manually remove the map value and drive the - // iterator. Defer delete connection container to avoid race condition in destroying - // connection. - dispatcher().deferredDelete(std::move(iter->second)); - connections_by_context_.erase(iter); - } - } - is_deleting_ = was_deleting; - } - -protected: - void cleanupConnections() { - is_deleting_ = true; - - // Purge sockets that have not progressed to connections. This should only happen when - // a listener filter stops iteration and never resumes. - while (!sockets_.empty()) { - auto removed = sockets_.front()->removeFromList(sockets_); - dispatcher().deferredDelete(std::move(removed)); - } - - for (auto& chain_and_connections : connections_by_context_) { - ASSERT(chain_and_connections.second != nullptr); - auto& connections = chain_and_connections.second->connections_; + void deferRemoveFilterChain(const Network::FilterChain* filter_chain) override { + auto iter = connections_by_context_.find(filter_chain); + if (iter == connections_by_context_.end()) { + // It is possible when listener is stopping. + } else { + auto& connections = iter->second->connections_; while (!connections.empty()) { connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); } + // Since is_deleting_ is on, we need to manually remove the map value and drive the + // iterator. Defer delete connection container to avoid race condition in destroying + // connection. + dispatcher().deferredDelete(std::move(iter->second)); + connections_by_context_.erase(iter); } - dispatcher().clearDeferredDeleteList(); } - absl::node_hash_map connections_by_context_; - bool is_deleting_{false}; +protected: + absl::node_hash_map + connections_by_context_; }; } // namespace Server From c1f79210c87e40c5cd09fa95c955b200f9c403d7 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 17 Jun 2021 09:01:48 +0000 Subject: [PATCH 13/23] push down conn collection or active conn op and push up Network::Connection op Signed-off-by: Yuchen Dai --- source/server/active_tcp_listener.cc | 46 +++------------------------- source/server/active_tcp_listener.h | 10 +++--- source/server/active_tcp_socket.h | 45 +++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 49 deletions(-) diff --git a/source/server/active_tcp_listener.cc b/source/server/active_tcp_listener.cc index 048820fbd4f9e..d8f9ca14b5df7 100644 --- a/source/server/active_tcp_listener.cc +++ b/source/server/active_tcp_listener.cc @@ -151,49 +151,13 @@ void ActiveTcpListener::resumeListening() { } } -void ActiveTcpListener::newConnection(Network::ConnectionSocketPtr&& socket, - std::unique_ptr stream_info) { - - // Find matching filter chain. - const auto filter_chain = config_->filterChainManager().findFilterChain(*socket); - if (filter_chain == nullptr) { - RELEASE_ASSERT(socket->addressProvider().remoteAddress() != nullptr, ""); - ENVOY_LOG(debug, "closing connection from {}: no matching filter chain found", - socket->addressProvider().remoteAddress()->asString()); - stats_.no_filter_chain_match_.inc(); - stream_info->setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); - stream_info->setResponseCodeDetails(StreamInfo::ResponseCodeDetails::get().FilterChainNotFound); - emitLogs(*config_, *stream_info); - socket->close(); - return; - } - - stream_info->setFilterChainName(filter_chain->name()); - auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); - stream_info->setDownstreamSslConnection(transport_socket->ssl()); - auto& active_connections = getOrCreateActiveConnections(*filter_chain); - auto server_conn_ptr = dispatcher().createServerConnection( - std::move(socket), std::move(transport_socket), *stream_info); - if (const auto timeout = filter_chain->transportSocketConnectTimeout(); - timeout != std::chrono::milliseconds::zero()) { - server_conn_ptr->setTransportSocketConnectTimeout(timeout); - } - - ActiveTcpConnectionPtr active_connection( +void ActiveTcpListener::newActiveConnection(const Network::FilterChain& filter_chain, + Network::ServerConnectionPtr server_conn_ptr, + std::unique_ptr stream_info) { + auto& active_connections = getOrCreateActiveConnections(filter_chain); + ActiveConnectionPtr active_connection( new ActiveTcpConnection(active_connections, std::move(server_conn_ptr), dispatcher().timeSource(), std::move(stream_info))); - active_connection->connection_->setBufferLimits(config_->perConnectionBufferLimitBytes()); - - RELEASE_ASSERT(active_connection->connection_->addressProvider().remoteAddress() != nullptr, ""); - - const bool empty_filter_chain = !config_->filterChainFactory().createNetworkFilterChain( - *active_connection->connection_, filter_chain->networkFilterFactories()); - if (empty_filter_chain) { - ENVOY_CONN_LOG(debug, "closing connection from {}: no filters", *active_connection->connection_, - active_connection->connection_->addressProvider().remoteAddress()->asString()); - active_connection->connection_->close(Network::ConnectionCloseType::NoFlush); - } - // If the connection is already closed, we can just let this connection immediately die. if (active_connection->connection_->state() != Network::Connection::State::Closed) { ENVOY_CONN_LOG(debug, "new connection from {}", *active_connection->connection_, diff --git a/source/server/active_tcp_listener.h b/source/server/active_tcp_listener.h index a62855b6115de..9df2549531a1b 100644 --- a/source/server/active_tcp_listener.h +++ b/source/server/active_tcp_listener.h @@ -70,8 +70,7 @@ struct ActiveTcpConnection : LinkedObject, */ class ActiveTcpListener final : public Network::TcpListenerCallbacks, public TypedActiveStreamListenerBase, - public Network::BalancedConnectionHandler, - Logger::Loggable { + public Network::BalancedConnectionHandler { public: ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerConfig& config); ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerPtr&& listener, @@ -119,10 +118,11 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, void removeConnection(ActiveTcpConnection& connection); /** - * Create a new connection from a socket accepted by the listener. + * Create active connection from the server connection. */ - void newConnection(Network::ConnectionSocketPtr&& socket, - std::unique_ptr stream_info) override; + void newActiveConnection(const Network::FilterChain& filter_chain, + Network::ServerConnectionPtr server_conn_ptr, + std::unique_ptr stream_info) override; /** * Return the active connections container attached with the given filter chain. diff --git a/source/server/active_tcp_socket.h b/source/server/active_tcp_socket.h index 9424009a1ca3a..f7b1c1e0e15a5 100644 --- a/source/server/active_tcp_socket.h +++ b/source/server/active_tcp_socket.h @@ -106,7 +106,8 @@ struct ActiveTcpSocket : public Network::ListenerFilterManager, bool connected_{false}; }; -class ActiveStreamListenerBase : public ActiveListenerImplBase { +class ActiveStreamListenerBase : public ActiveListenerImplBase, + protected Logger::Loggable { public: ActiveStreamListenerBase(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, Network::ListenerPtr&& listener, Network::ListenerConfig& config); @@ -135,8 +136,46 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase { /** * Create a new connection from a socket accepted by the listener. */ - virtual void newConnection(Network::ConnectionSocketPtr&& socket, - std::unique_ptr stream_info) PURE; + void newConnection(Network::ConnectionSocketPtr&& socket, + std::unique_ptr stream_info) { + // Find matching filter chain. + const auto filter_chain = config_->filterChainManager().findFilterChain(*socket); + if (filter_chain == nullptr) { + RELEASE_ASSERT(socket->addressProvider().remoteAddress() != nullptr, ""); + ENVOY_LOG(debug, "closing connection from {}: no matching filter chain found", + socket->addressProvider().remoteAddress()->asString()); + stats_.no_filter_chain_match_.inc(); + stream_info->setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); + stream_info->setResponseCodeDetails( + StreamInfo::ResponseCodeDetails::get().FilterChainNotFound); + emitLogs(*config_, *stream_info); + socket->close(); + return; + } + stream_info->setFilterChainName(filter_chain->name()); + auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); + stream_info->setDownstreamSslConnection(transport_socket->ssl()); + auto server_conn_ptr = dispatcher().createServerConnection( + std::move(socket), std::move(transport_socket), *stream_info); + if (const auto timeout = filter_chain->transportSocketConnectTimeout(); + timeout != std::chrono::milliseconds::zero()) { + server_conn_ptr->setTransportSocketConnectTimeout(timeout); + } + server_conn_ptr->setBufferLimits(config_->perConnectionBufferLimitBytes()); + RELEASE_ASSERT(server_conn_ptr->addressProvider().remoteAddress() != nullptr, ""); + const bool empty_filter_chain = !config_->filterChainFactory().createNetworkFilterChain( + *server_conn_ptr, filter_chain->networkFilterFactories()); + if (empty_filter_chain) { + ENVOY_CONN_LOG(debug, "closing connection from {}: no filters", *server_conn_ptr, + server_conn_ptr->addressProvider().remoteAddress()->asString()); + server_conn_ptr->close(Network::ConnectionCloseType::NoFlush); + } + newActiveConnection(*filter_chain, std::move(server_conn_ptr), std::move(stream_info)); + } + + virtual void newActiveConnection(const Network::FilterChain& filter_chain, + Network::ServerConnectionPtr server_conn_ptr, + std::unique_ptr stream_info) PURE; /** * Schedule to remove and destroy the active connections owned by the filter chain. From bbee97911de7505a4e5e1b31b23902df8af8e00a Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 1 Jul 2021 22:55:39 +0000 Subject: [PATCH 14/23] tmpaddinternallistener_pre_1 Signed-off-by: Yuchen Dai --- source/server/active_tcp_socket.cc | 8 +++++--- source/server/active_tcp_socket.h | 20 +++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/source/server/active_tcp_socket.cc b/source/server/active_tcp_socket.cc index 89761ce347689..fbfd0db580b5d 100644 --- a/source/server/active_tcp_socket.cc +++ b/source/server/active_tcp_socket.cc @@ -42,9 +42,11 @@ ActiveStreamListenerBase::ActiveStreamListenerBase(Network::ConnectionHandler& p Event::Dispatcher& dispatcher, Network::ListenerPtr&& listener, Network::ListenerConfig& config) - : ActiveListenerImplBase(parent, &config), parent_(parent), dispatcher_(dispatcher), - listener_(std::move(listener)), listener_filters_timeout_(config.listenerFiltersTimeout()), - continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()) {} + : ActiveListenerImplBase(parent, &config), parent_(parent), + listener_filters_timeout_(config.listenerFiltersTimeout()), + continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()), + dispatcher_(dispatcher), + listener_(std::move(listener)) {} void ActiveStreamListenerBase::emitLogs(Network::ListenerConfig& config, StreamInfo::StreamInfo& stream_info) { diff --git a/source/server/active_tcp_socket.h b/source/server/active_tcp_socket.h index f7b1c1e0e15a5..59d50ea1c9960 100644 --- a/source/server/active_tcp_socket.h +++ b/source/server/active_tcp_socket.h @@ -173,9 +173,6 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, newActiveConnection(*filter_chain, std::move(server_conn_ptr), std::move(stream_info)); } - virtual void newActiveConnection(const Network::FilterChain& filter_chain, - Network::ServerConnectionPtr server_conn_ptr, - std::unique_ptr stream_info) PURE; /** * Schedule to remove and destroy the active connections owned by the filter chain. @@ -210,12 +207,25 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, } } + // Below members are open to access by ActiveTcpSocket. Network::ConnectionHandler& parent_; - Event::Dispatcher& dispatcher_; - Network::ListenerPtr listener_; const std::chrono::milliseconds listener_filters_timeout_; const bool continue_on_listener_filters_timeout_; std::list> sockets_; + +protected: + /** + * Create the active connection from server connection. This active listener owns the created active connection. + * + * @param filter_chain The network filter chain linking to the connection. + * @param server_conn_ptr The server connection. + * @param stream_info The stream info of the active connection. + */ + virtual void newActiveConnection(const Network::FilterChain& filter_chain, + Network::ServerConnectionPtr server_conn_ptr, + std::unique_ptr stream_info) PURE; + Event::Dispatcher& dispatcher_; + Network::ListenerPtr listener_; bool is_deleting_{false}; }; From 57ca809ebe91f8d81214299f939edf82afbfcd20 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 2 Jul 2021 00:36:46 +0000 Subject: [PATCH 15/23] fix format Signed-off-by: Yuchen Dai --- source/server/active_tcp_socket.cc | 5 ++--- source/server/active_tcp_socket.h | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/source/server/active_tcp_socket.cc b/source/server/active_tcp_socket.cc index fbfd0db580b5d..a803edb857806 100644 --- a/source/server/active_tcp_socket.cc +++ b/source/server/active_tcp_socket.cc @@ -42,11 +42,10 @@ ActiveStreamListenerBase::ActiveStreamListenerBase(Network::ConnectionHandler& p Event::Dispatcher& dispatcher, Network::ListenerPtr&& listener, Network::ListenerConfig& config) - : ActiveListenerImplBase(parent, &config), parent_(parent), + : ActiveListenerImplBase(parent, &config), parent_(parent), listener_filters_timeout_(config.listenerFiltersTimeout()), continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()), - dispatcher_(dispatcher), - listener_(std::move(listener)) {} + dispatcher_(dispatcher), listener_(std::move(listener)) {} void ActiveStreamListenerBase::emitLogs(Network::ListenerConfig& config, StreamInfo::StreamInfo& stream_info) { diff --git a/source/server/active_tcp_socket.h b/source/server/active_tcp_socket.h index 59d50ea1c9960..46bf6d76e9002 100644 --- a/source/server/active_tcp_socket.h +++ b/source/server/active_tcp_socket.h @@ -173,7 +173,6 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, newActiveConnection(*filter_chain, std::move(server_conn_ptr), std::move(stream_info)); } - /** * Schedule to remove and destroy the active connections owned by the filter chain. */ @@ -215,8 +214,9 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, protected: /** - * Create the active connection from server connection. This active listener owns the created active connection. - * + * Create the active connection from server connection. This active listener owns the created + * active connection. + * * @param filter_chain The network filter chain linking to the connection. * @param server_conn_ptr The server connection. * @param stream_info The stream info of the active connection. From 4523c99f192ab3c4ac0da7c78d233370bec08c75 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 2 Jul 2021 20:38:58 +0000 Subject: [PATCH 16/23] address comment Signed-off-by: Yuchen Dai --- source/server/active_tcp_listener.cc | 19 ------------------- source/server/active_tcp_listener.h | 6 ------ source/server/active_tcp_socket.h | 26 ++++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/source/server/active_tcp_listener.cc b/source/server/active_tcp_listener.cc index d8f9ca14b5df7..3b5e63cdc6e21 100644 --- a/source/server/active_tcp_listener.cc +++ b/source/server/active_tcp_listener.cc @@ -67,25 +67,6 @@ ActiveTcpListener::~ActiveTcpListener() { config_->name(), numConnections())); } -void ActiveTcpListener::removeConnection(ActiveTcpConnection& connection) { - ENVOY_CONN_LOG(debug, "adding to cleanup list", *connection.connection_); - ActiveConnections& active_connections = connection.active_connections_; - ActiveTcpConnectionPtr removed = connection.removeFromList(active_connections.connections_); - dispatcher().deferredDelete(std::move(removed)); - // Delete map entry only iff connections becomes empty. - if (active_connections.connections_.empty()) { - auto iter = connections_by_context_.find(&active_connections.filter_chain_); - ASSERT(iter != connections_by_context_.end()); - // To cover the lifetime of every single connection, Connections need to be deferred deleted - // because the previously contained connection is deferred deleted. - dispatcher().deferredDelete(std::move(iter->second)); - // The erase will break the iteration over the connections_by_context_ during the deletion. - if (!is_deleting_) { - connections_by_context_.erase(iter); - } - } -} - void ActiveTcpListener::updateListenerConfig(Network::ListenerConfig& config) { ENVOY_LOG(trace, "replacing listener ", config_->listenerTag(), " by ", config.listenerTag()); ASSERT(&config_->connectionBalancer() == &config.connectionBalancer()); diff --git a/source/server/active_tcp_listener.h b/source/server/active_tcp_listener.h index 9df2549531a1b..95e4da38197d9 100644 --- a/source/server/active_tcp_listener.h +++ b/source/server/active_tcp_listener.h @@ -111,12 +111,6 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, void onAcceptWorker(Network::ConnectionSocketPtr&& socket, bool hand_off_restored_destination_connections, bool rebalanced) override; - /** - * Remove and destroy an active connection. - * @param connection supplies the connection to remove. - */ - void removeConnection(ActiveTcpConnection& connection); - /** * Create active connection from the server connection. */ diff --git a/source/server/active_tcp_socket.h b/source/server/active_tcp_socket.h index 46bf6d76e9002..932b8df37603c 100644 --- a/source/server/active_tcp_socket.h +++ b/source/server/active_tcp_socket.h @@ -240,6 +240,9 @@ class TypedActiveStreamListenerBase : public ActiveStreamListenerBase { using ActiveConnectionPtr = std::unique_ptr; using ActiveConnectionCollectionPtr = std::unique_ptr; + /** + * Schedule to remove and destroy the active connection owned by the filter chain. + */ void deferRemoveFilterChain(const Network::FilterChain* filter_chain) override { auto iter = connections_by_context_.find(filter_chain); if (iter == connections_by_context_.end()) { @@ -257,6 +260,29 @@ class TypedActiveStreamListenerBase : public ActiveStreamListenerBase { } } + /** + * Remove and destroy an active connection. + * @param connection supplies the connection to remove. + */ + void removeConnection(ActiveConnectionType& connection) { + ENVOY_CONN_LOG(debug, "adding to cleanup list", *connection.connection_); + ActiveConnectionCollectionType& active_connections = connection.active_connections_; + ActiveConnectionPtr removed = connection.removeFromList(active_connections.connections_); + dispatcher().deferredDelete(std::move(removed)); + // Delete map entry only iff connections becomes empty. + if (active_connections.connections_.empty()) { + auto iter = connections_by_context_.find(&active_connections.filter_chain_); + ASSERT(iter != connections_by_context_.end()); + // To cover the lifetime of every single connection, Connections need to be deferred deleted + // because the previously contained connection is deferred deleted. + dispatcher().deferredDelete(std::move(iter->second)); + // The erase will break the iteration over the connections_by_context_ during the deletion. + if (!is_deleting_) { + connections_by_context_.erase(iter); + } + } + } + protected: absl::node_hash_map connections_by_context_; From dfe40e4698146c1e7a338eea8c2ae44fd91615ce Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 15 Jul 2021 00:14:31 +0000 Subject: [PATCH 17/23] listener: extract active_tcp_socket and active_stream_listener_base Signed-off-by: Yuchen Dai --- source/server/BUILD | 61 +++++++-- source/server/active_tcp_listener.cc | 56 ++++++-- source/server/active_tcp_listener.h | 101 ++++++++------ source/server/active_tcp_socket.cc | 21 +-- source/server/active_tcp_socket.h | 192 +-------------------------- 5 files changed, 153 insertions(+), 278 deletions(-) diff --git a/source/server/BUILD b/source/server/BUILD index 10699ee777ba6..8155f332fb7e7 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -116,26 +116,20 @@ envoy_cc_library( "active_tcp_listener.h", ], deps = [ + ":active_stream_listener_base", ":active_tcp_socket", "//envoy/common:time_interface", "//envoy/event:deferred_deletable", "//envoy/event:dispatcher_interface", - "//envoy/event:timer_interface", "//envoy/network:connection_handler_interface", "//envoy/network:connection_interface", - "//envoy/network:filter_interface", "//envoy/network:listen_socket_interface", "//envoy/network:listener_interface", "//envoy/server:listener_manager_interface", - "//envoy/stats:timespan_interface", "//source/common/common:assert_lib", "//source/common/common:linked_object", - "//source/common/common:non_copyable", - "//source/common/common:safe_memcpy_lib", - "//source/common/event:deferred_task", "//source/common/network:connection_lib", "//source/common/stats:timespan_lib", - "//source/common/stream_info:stream_info_lib", "//source/server:active_listener_base", ], ) @@ -158,6 +152,28 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "active_tcp_listener_headers", + hdrs = [ + "active_stream_listener_base.h", + "active_tcp_socket.h", + ], + deps = [ + ":active_listener_base", + "//envoy/common:time_interface", + "//envoy/event:deferred_deletable", + "//envoy/event:dispatcher_interface", + "//envoy/event:timer_interface", + "//envoy/network:connection_handler_interface", + "//envoy/network:connection_interface", + "//envoy/network:filter_interface", + "//envoy/network:listen_socket_interface", + "//envoy/network:listener_interface", + "//envoy/server:listener_manager_interface", + "//source/common/common:linked_object", + ], +) + envoy_cc_library( name = "active_tcp_socket", srcs = ["active_tcp_socket.cc"], @@ -166,27 +182,46 @@ envoy_cc_library( ], deps = [ ":active_listener_base", + ":active_tcp_listener_headers", "//envoy/common:time_interface", "//envoy/event:deferred_deletable", "//envoy/event:dispatcher_interface", "//envoy/event:timer_interface", "//envoy/network:connection_handler_interface", "//envoy/network:connection_interface", - "//envoy/network:exception_interface", "//envoy/network:filter_interface", "//envoy/network:listen_socket_interface", "//envoy/network:listener_interface", - "//envoy/server:listener_manager_interface", - "//envoy/stats:timespan_interface", "//source/common/common:linked_object", - "//source/common/common:non_copyable", - "//source/common/event:deferred_task", "//source/common/network:connection_lib", - "//source/common/stats:timespan_lib", "//source/common/stream_info:stream_info_lib", ], ) +envoy_cc_library( + name = "active_stream_listener_base", + srcs = ["active_stream_listener_base.cc"], + hdrs = [ + "active_stream_listener_base.h", + ], + deps = [ + ":active_listener_base", + ":active_tcp_listener_headers", + "//envoy/common:time_interface", + "//envoy/event:deferred_deletable", + "//envoy/event:dispatcher_interface", + "//envoy/event:timer_interface", + "//envoy/network:connection_handler_interface", + "//envoy/network:connection_interface", + "//envoy/network:filter_interface", + "//envoy/network:listen_socket_interface", + "//envoy/network:listener_interface", + "//envoy/stream_info:stream_info_interface", + "//source/common/common:linked_object", + "//source/common/network:connection_lib", + ], +) + envoy_cc_library( name = "drain_manager_lib", srcs = ["drain_manager_impl.cc"], diff --git a/source/server/active_tcp_listener.cc b/source/server/active_tcp_listener.cc index 3b5e63cdc6e21..c843f8aed8c53 100644 --- a/source/server/active_tcp_listener.cc +++ b/source/server/active_tcp_listener.cc @@ -3,12 +3,9 @@ #include #include "envoy/event/dispatcher.h" -#include "envoy/event/timer.h" -#include "envoy/network/filter.h" #include "envoy/stats/scope.h" #include "source/common/common/assert.h" -#include "source/common/event/deferred_task.h" #include "source/common/network/connection_impl.h" #include "source/common/network/utility.h" #include "source/common/stats/timespan_impl.h" @@ -18,7 +15,7 @@ namespace Server { ActiveTcpListener::ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerConfig& config) - : TypedActiveStreamListenerBase( + : ActiveStreamListenerBase( parent, parent.dispatcher(), parent.dispatcher().createListener(config.listenSocketFactory().getListenSocket(), *this, config.bindToPort(), config.tcpBacklogSize()), @@ -30,8 +27,7 @@ ActiveTcpListener::ActiveTcpListener(Network::TcpConnectionHandler& parent, ActiveTcpListener::ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerPtr&& listener, Network::ListenerConfig& config) - : TypedActiveStreamListenerBase(parent, parent.dispatcher(), - std::move(listener), config), + : ActiveStreamListenerBase(parent, parent.dispatcher(), std::move(listener), config), tcp_conn_handler_(parent) { config.connectionBalancer().registerHandler(*this); } @@ -67,12 +63,48 @@ ActiveTcpListener::~ActiveTcpListener() { config_->name(), numConnections())); } +void ActiveTcpListener::removeConnection(ActiveTcpConnection& connection) { + ENVOY_CONN_LOG(debug, "adding to cleanup list", *connection.connection_); + ActiveConnections& active_connections = connection.active_connections_; + auto removed = connection.removeFromList(active_connections.connections_); + dispatcher().deferredDelete(std::move(removed)); + // Delete map entry only iff connections becomes empty. + if (active_connections.connections_.empty()) { + auto iter = connections_by_context_.find(&active_connections.filter_chain_); + ASSERT(iter != connections_by_context_.end()); + // To cover the lifetime of every single connection, Connections need to be deferred deleted + // because the previously contained connection is deferred deleted. + dispatcher().deferredDelete(std::move(iter->second)); + // The erase will break the iteration over the connections_by_context_ during the deletion. + if (!is_deleting_) { + connections_by_context_.erase(iter); + } + } +} + void ActiveTcpListener::updateListenerConfig(Network::ListenerConfig& config) { ENVOY_LOG(trace, "replacing listener ", config_->listenerTag(), " by ", config.listenerTag()); ASSERT(&config_->connectionBalancer() == &config.connectionBalancer()); config_ = &config; } +void ActiveTcpListener::deferRemoveFilterChain(const Network::FilterChain* filter_chain) { + auto iter = connections_by_context_.find(filter_chain); + if (iter == connections_by_context_.end()) { + // It is possible when listener is stopping. + } else { + auto& connections = iter->second->connections_; + while (!connections.empty()) { + connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); + } + // Since is_deleting_ is on, we need to manually remove the map value and drive the + // iterator. Defer delete connection container to avoid race condition in destroying + // connection. + dispatcher().deferredDelete(std::move(iter->second)); + connections_by_context_.erase(iter); + } +} + void ActiveTcpListener::onAccept(Network::ConnectionSocketPtr&& socket) { if (listenerConnectionLimitReached()) { RELEASE_ASSERT(socket->addressProvider().remoteAddress() != nullptr, ""); @@ -115,11 +147,6 @@ void ActiveTcpListener::onAcceptWorker(Network::ConnectionSocketPtr&& socket, onSocketAccepted(std::move(active_socket)); } -Network::BalancedConnectionHandlerOptRef -ActiveTcpListener::getBalancedHandlerByAddress(const Network::Address::Instance& address) { - return tcp_conn_handler_.getBalancedHandlerByAddress(address); -} - void ActiveTcpListener::pauseListening() { if (listener_ != nullptr) { listener_->disable(); @@ -132,11 +159,16 @@ void ActiveTcpListener::resumeListening() { } } +Network::BalancedConnectionHandlerOptRef +ActiveTcpListener::getBalancedHandlerByAddress(const Network::Address::Instance& address) { + return tcp_conn_handler_.getBalancedHandlerByAddress(address); +} + void ActiveTcpListener::newActiveConnection(const Network::FilterChain& filter_chain, Network::ServerConnectionPtr server_conn_ptr, std::unique_ptr stream_info) { auto& active_connections = getOrCreateActiveConnections(filter_chain); - ActiveConnectionPtr active_connection( + ActiveTcpConnectionPtr active_connection( new ActiveTcpConnection(active_connections, std::move(server_conn_ptr), dispatcher().timeSource(), std::move(stream_info))); // If the connection is already closed, we can just let this connection immediately die. diff --git a/source/server/active_tcp_listener.h b/source/server/active_tcp_listener.h index 95e4da38197d9..67d4049951456 100644 --- a/source/server/active_tcp_listener.h +++ b/source/server/active_tcp_listener.h @@ -1,14 +1,16 @@ #pragma once #include "envoy/event/dispatcher.h" -#include "envoy/event/timer.h" #include "envoy/stats/timespan.h" +#include "envoy/stream_info/stream_info.h" #include "source/common/common/linked_object.h" -#include "source/common/stream_info/stream_info_impl.h" #include "source/server/active_listener_base.h" +#include "source/server/active_stream_listener_base.h" #include "source/server/active_tcp_socket.h" +#include "absl/container/node_hash_map.h" + namespace Envoy { namespace Server { @@ -17,8 +19,6 @@ using ActiveTcpConnectionPtr = std::unique_ptr; class ActiveConnections; using ActiveConnectionCollectionPtr = std::unique_ptr; -class ActiveTcpListener; - namespace { // Structure used to allow a unique_ptr to be captured in a posted lambda. See below. struct RebalancedSocket { @@ -27,55 +27,18 @@ struct RebalancedSocket { using RebalancedSocketSharedPtr = std::shared_ptr; } // namespace -/** - * Wrapper for a group of active connections which are attached to the same filter chain context. - */ -class ActiveConnections : public Event::DeferredDeletable { -public: - ActiveConnections(ActiveTcpListener& listener, const Network::FilterChain& filter_chain); - ~ActiveConnections() override; - - // listener filter chain pair is the owner of the connections - ActiveTcpListener& listener_; - const Network::FilterChain& filter_chain_; - // Owned connections - std::list connections_; -}; - -/** - * Wrapper for an active TCP connection owned by this handler. - */ -struct ActiveTcpConnection : LinkedObject, - public Event::DeferredDeletable, - public Network::ConnectionCallbacks, - Logger::Loggable { - ActiveTcpConnection(ActiveConnections& active_connections, - Network::ConnectionPtr&& new_connection, TimeSource& time_system, - std::unique_ptr&& stream_info); - ~ActiveTcpConnection() override; - using CollectionType = ActiveConnections; - // Network::ConnectionCallbacks - void onEvent(Network::ConnectionEvent event) override; - void onAboveWriteBufferHighWatermark() override {} - void onBelowWriteBufferLowWatermark() override {} - - std::unique_ptr stream_info_; - ActiveConnections& active_connections_; - Network::ConnectionPtr connection_; - Stats::TimespanPtr conn_length_; -}; - /** * Wrapper for an active tcp listener owned by this handler. */ class ActiveTcpListener final : public Network::TcpListenerCallbacks, - public TypedActiveStreamListenerBase, + public ActiveStreamListenerBase, public Network::BalancedConnectionHandler { public: ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerConfig& config); ActiveTcpListener(Network::TcpConnectionHandler& parent, Network::ListenerPtr&& listener, Network::ListenerConfig& config); ~ActiveTcpListener() override; + bool listenerConnectionLimitReached() const { // TODO(tonya11en): Delegate enforcement of per-listener connection limits to overload // manager. @@ -129,6 +92,20 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, */ void updateListenerConfig(Network::ListenerConfig& config); + /** + * Schedule to remove and destroy the active connection owned by the filter chain. + */ + void deferRemoveFilterChain(const Network::FilterChain* filter_chain) override; + + /** + * Remove and destroy an active connection. + * @param connection supplies the connection to remove. + */ + void removeConnection(ActiveTcpConnection& connection); + + absl::node_hash_map> + connections_by_context_; + Network::TcpConnectionHandler& tcp_conn_handler_; // The number of connections currently active on this listener. This is typically used for // connection balancing across per-handler listeners. @@ -137,5 +114,43 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, using ActiveTcpListenerOptRef = absl::optional>; +/** + * Wrapper for a group of active connections which are attached to the same filter chain context. + */ +class ActiveConnections : public Event::DeferredDeletable { +public: + ActiveConnections(ActiveTcpListener& listener, const Network::FilterChain& filter_chain); + ~ActiveConnections() override; + + // listener filter chain pair is the owner of the connections + ActiveTcpListener& listener_; + const Network::FilterChain& filter_chain_; + // Owned connections + std::list connections_; +}; + +/** + * Wrapper for an active TCP connection owned by this handler. + */ +struct ActiveTcpConnection : LinkedObject, + public Event::DeferredDeletable, + public Network::ConnectionCallbacks, + Logger::Loggable { + ActiveTcpConnection(ActiveConnections& active_connections, + Network::ConnectionPtr&& new_connection, TimeSource& time_system, + std::unique_ptr&& stream_info); + ~ActiveTcpConnection() override; + + // Network::ConnectionCallbacks + void onEvent(Network::ConnectionEvent event) override; + void onAboveWriteBufferHighWatermark() override {} + void onBelowWriteBufferLowWatermark() override {} + + std::unique_ptr stream_info_; + ActiveConnections& active_connections_; + Network::ConnectionPtr connection_; + Stats::TimespanPtr conn_length_; +}; + } // namespace Server } // namespace Envoy diff --git a/source/server/active_tcp_socket.cc b/source/server/active_tcp_socket.cc index a418bdcbddaf4..33740b957a42d 100644 --- a/source/server/active_tcp_socket.cc +++ b/source/server/active_tcp_socket.cc @@ -1,9 +1,9 @@ #include "source/server/active_tcp_socket.h" #include "envoy/network/filter.h" -#include "envoy/stats/scope.h" -#include "source/common/stats/timespan_impl.h" +#include "source/common/stream_info/stream_info_impl.h" +#include "source/server/active_stream_listener_base.h" namespace Envoy { namespace Server { @@ -38,23 +38,6 @@ ActiveTcpSocket::~ActiveTcpSocket() { Event::Dispatcher& ActiveTcpSocket::dispatcher() { return listener_.dispatcher(); } -ActiveStreamListenerBase::ActiveStreamListenerBase(Network::ConnectionHandler& parent, - Event::Dispatcher& dispatcher, - Network::ListenerPtr&& listener, - Network::ListenerConfig& config) - : ActiveListenerImplBase(parent, &config), parent_(parent), - listener_filters_timeout_(config.listenerFiltersTimeout()), - continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()), - dispatcher_(dispatcher), listener_(std::move(listener)) {} - -void ActiveStreamListenerBase::emitLogs(Network::ListenerConfig& config, - StreamInfo::StreamInfo& stream_info) { - stream_info.onRequestComplete(); - for (const auto& access_log : config.accessLogs()) { - access_log->log(nullptr, nullptr, nullptr, stream_info); - } -} - void ActiveTcpSocket::onTimeout() { listener_.stats_.downstream_pre_cx_timeout_.inc(); ASSERT(inserted()); diff --git a/source/server/active_tcp_socket.h b/source/server/active_tcp_socket.h index 932b8df37603c..9900bbd66b883 100644 --- a/source/server/active_tcp_socket.h +++ b/source/server/active_tcp_socket.h @@ -8,22 +8,13 @@ #include "envoy/common/time.h" #include "envoy/event/deferred_deletable.h" #include "envoy/event/dispatcher.h" -#include "envoy/network/connection.h" -#include "envoy/network/connection_handler.h" #include "envoy/network/filter.h" #include "envoy/network/listen_socket.h" #include "envoy/network/listener.h" -#include "envoy/server/listener_manager.h" -#include "envoy/stats/scope.h" -#include "envoy/stats/timespan.h" #include "source/common/common/linked_object.h" -#include "source/common/common/non_copyable.h" -#include "source/common/stream_info/stream_info_impl.h" #include "source/server/active_listener_base.h" -#include "spdlog/spdlog.h" - namespace Envoy { namespace Server { @@ -96,6 +87,7 @@ struct ActiveTcpSocket : public Network::ListenerFilterManager, StreamInfo::FilterState& filterState() override { return *stream_info_->filterState().get(); } + // The owner of this ActiveTcpSocket. ActiveStreamListenerBase& listener_; Network::ConnectionSocketPtr socket_; const bool hand_off_restored_destination_connections_; @@ -106,187 +98,5 @@ struct ActiveTcpSocket : public Network::ListenerFilterManager, bool connected_{false}; }; -class ActiveStreamListenerBase : public ActiveListenerImplBase, - protected Logger::Loggable { -public: - ActiveStreamListenerBase(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, - Network::ListenerPtr&& listener, Network::ListenerConfig& config); - static void emitLogs(Network::ListenerConfig& config, StreamInfo::StreamInfo& stream_info); - - Event::Dispatcher& dispatcher() { return dispatcher_; } - - /** - * Schedule to remove and destroy the active connections which are not tracked by listener - * config. Caution: The connection are not destroyed yet when function returns. - */ - void - deferredRemoveFilterChains(const std::list& draining_filter_chains) { - // Need to recover the original deleting state. - const bool was_deleting = is_deleting_; - is_deleting_ = true; - for (const auto* filter_chain : draining_filter_chains) { - deferRemoveFilterChain(filter_chain); - } - is_deleting_ = was_deleting; - } - - virtual void incNumConnections() PURE; - virtual void decNumConnections() PURE; - - /** - * Create a new connection from a socket accepted by the listener. - */ - void newConnection(Network::ConnectionSocketPtr&& socket, - std::unique_ptr stream_info) { - // Find matching filter chain. - const auto filter_chain = config_->filterChainManager().findFilterChain(*socket); - if (filter_chain == nullptr) { - RELEASE_ASSERT(socket->addressProvider().remoteAddress() != nullptr, ""); - ENVOY_LOG(debug, "closing connection from {}: no matching filter chain found", - socket->addressProvider().remoteAddress()->asString()); - stats_.no_filter_chain_match_.inc(); - stream_info->setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); - stream_info->setResponseCodeDetails( - StreamInfo::ResponseCodeDetails::get().FilterChainNotFound); - emitLogs(*config_, *stream_info); - socket->close(); - return; - } - stream_info->setFilterChainName(filter_chain->name()); - auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); - stream_info->setDownstreamSslConnection(transport_socket->ssl()); - auto server_conn_ptr = dispatcher().createServerConnection( - std::move(socket), std::move(transport_socket), *stream_info); - if (const auto timeout = filter_chain->transportSocketConnectTimeout(); - timeout != std::chrono::milliseconds::zero()) { - server_conn_ptr->setTransportSocketConnectTimeout(timeout); - } - server_conn_ptr->setBufferLimits(config_->perConnectionBufferLimitBytes()); - RELEASE_ASSERT(server_conn_ptr->addressProvider().remoteAddress() != nullptr, ""); - const bool empty_filter_chain = !config_->filterChainFactory().createNetworkFilterChain( - *server_conn_ptr, filter_chain->networkFilterFactories()); - if (empty_filter_chain) { - ENVOY_CONN_LOG(debug, "closing connection from {}: no filters", *server_conn_ptr, - server_conn_ptr->addressProvider().remoteAddress()->asString()); - server_conn_ptr->close(Network::ConnectionCloseType::NoFlush); - } - newActiveConnection(*filter_chain, std::move(server_conn_ptr), std::move(stream_info)); - } - - /** - * Schedule to remove and destroy the active connections owned by the filter chain. - */ - virtual void deferRemoveFilterChain(const Network::FilterChain* filter_chain) PURE; - - virtual Network::BalancedConnectionHandlerOptRef - getBalancedHandlerByAddress(const Network::Address::Instance& address) PURE; - - void onSocketAccepted(std::unique_ptr active_socket) { - // Create and run the filters - config_->filterChainFactory().createListenerFilterChain(*active_socket); - active_socket->continueFilterChain(true); - - // Move active_socket to the sockets_ list if filter iteration needs to continue later. - // Otherwise we let active_socket be destructed when it goes out of scope. - if (active_socket->iter_ != active_socket->accept_filters_.end()) { - active_socket->startTimer(); - LinkedList::moveIntoListBack(std::move(active_socket), sockets_); - } else { - if (!active_socket->connected_) { - // If active_socket is about to be destructed, emit logs if a connection is not created. - if (active_socket->stream_info_ != nullptr) { - emitLogs(*config_, *active_socket->stream_info_); - } else { - // If the active_socket is not connected, this socket is not promoted to active - // connection. Thus the stream_info_ is owned by this active socket. - ENVOY_BUG(active_socket->stream_info_ != nullptr, - "the unconnected active socket must have stream info."); - } - } - } - } - - // Below members are open to access by ActiveTcpSocket. - Network::ConnectionHandler& parent_; - const std::chrono::milliseconds listener_filters_timeout_; - const bool continue_on_listener_filters_timeout_; - std::list> sockets_; - -protected: - /** - * Create the active connection from server connection. This active listener owns the created - * active connection. - * - * @param filter_chain The network filter chain linking to the connection. - * @param server_conn_ptr The server connection. - * @param stream_info The stream info of the active connection. - */ - virtual void newActiveConnection(const Network::FilterChain& filter_chain, - Network::ServerConnectionPtr server_conn_ptr, - std::unique_ptr stream_info) PURE; - Event::Dispatcher& dispatcher_; - Network::ListenerPtr listener_; - bool is_deleting_{false}; -}; - -template -class TypedActiveStreamListenerBase : public ActiveStreamListenerBase { - -public: - using ActiveConnectionCollectionType = typename ActiveConnectionType::CollectionType; - TypedActiveStreamListenerBase(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, - Network::ListenerPtr&& listener, Network::ListenerConfig& config) - : ActiveStreamListenerBase(parent, dispatcher, std::move(listener), config) {} - using ActiveConnectionPtr = std::unique_ptr; - using ActiveConnectionCollectionPtr = std::unique_ptr; - - /** - * Schedule to remove and destroy the active connection owned by the filter chain. - */ - void deferRemoveFilterChain(const Network::FilterChain* filter_chain) override { - auto iter = connections_by_context_.find(filter_chain); - if (iter == connections_by_context_.end()) { - // It is possible when listener is stopping. - } else { - auto& connections = iter->second->connections_; - while (!connections.empty()) { - connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); - } - // Since is_deleting_ is on, we need to manually remove the map value and drive the - // iterator. Defer delete connection container to avoid race condition in destroying - // connection. - dispatcher().deferredDelete(std::move(iter->second)); - connections_by_context_.erase(iter); - } - } - - /** - * Remove and destroy an active connection. - * @param connection supplies the connection to remove. - */ - void removeConnection(ActiveConnectionType& connection) { - ENVOY_CONN_LOG(debug, "adding to cleanup list", *connection.connection_); - ActiveConnectionCollectionType& active_connections = connection.active_connections_; - ActiveConnectionPtr removed = connection.removeFromList(active_connections.connections_); - dispatcher().deferredDelete(std::move(removed)); - // Delete map entry only iff connections becomes empty. - if (active_connections.connections_.empty()) { - auto iter = connections_by_context_.find(&active_connections.filter_chain_); - ASSERT(iter != connections_by_context_.end()); - // To cover the lifetime of every single connection, Connections need to be deferred deleted - // because the previously contained connection is deferred deleted. - dispatcher().deferredDelete(std::move(iter->second)); - // The erase will break the iteration over the connections_by_context_ during the deletion. - if (!is_deleting_) { - connections_by_context_.erase(iter); - } - } - } - -protected: - absl::node_hash_map - connections_by_context_; -}; - } // namespace Server } // namespace Envoy From a497d0b7d12d91f18a919cc2194985bc7c3aea1d Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 15 Jul 2021 00:25:28 +0000 Subject: [PATCH 18/23] add missing files Signed-off-by: Yuchen Dai --- source/server/active_stream_listener_base.cc | 62 ++++++++++ source/server/active_stream_listener_base.h | 115 +++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 source/server/active_stream_listener_base.cc create mode 100644 source/server/active_stream_listener_base.h diff --git a/source/server/active_stream_listener_base.cc b/source/server/active_stream_listener_base.cc new file mode 100644 index 0000000000000..1a3fe28233167 --- /dev/null +++ b/source/server/active_stream_listener_base.cc @@ -0,0 +1,62 @@ +#include "source/server/active_stream_listener_base.h" + +#include "envoy/network/filter.h" + +namespace Envoy { +namespace Server { + +ActiveStreamListenerBase::ActiveStreamListenerBase(Network::ConnectionHandler& parent, + Event::Dispatcher& dispatcher, + Network::ListenerPtr&& listener, + Network::ListenerConfig& config) + : ActiveListenerImplBase(parent, &config), parent_(parent), + listener_filters_timeout_(config.listenerFiltersTimeout()), + continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()), + dispatcher_(dispatcher), listener_(std::move(listener)) {} + +void ActiveStreamListenerBase::emitLogs(Network::ListenerConfig& config, + StreamInfo::StreamInfo& stream_info) { + stream_info.onRequestComplete(); + for (const auto& access_log : config.accessLogs()) { + access_log->log(nullptr, nullptr, nullptr, stream_info); + } +} + +void ActiveStreamListenerBase::newConnection(Network::ConnectionSocketPtr&& socket, + std::unique_ptr stream_info) { + // Find matching filter chain. + const auto filter_chain = config_->filterChainManager().findFilterChain(*socket); + if (filter_chain == nullptr) { + RELEASE_ASSERT(socket->addressProvider().remoteAddress() != nullptr, ""); + ENVOY_LOG(debug, "closing connection from {}: no matching filter chain found", + socket->addressProvider().remoteAddress()->asString()); + stats_.no_filter_chain_match_.inc(); + stream_info->setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); + stream_info->setResponseCodeDetails(StreamInfo::ResponseCodeDetails::get().FilterChainNotFound); + emitLogs(*config_, *stream_info); + socket->close(); + return; + } + stream_info->setFilterChainName(filter_chain->name()); + auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); + stream_info->setDownstreamSslConnection(transport_socket->ssl()); + auto server_conn_ptr = dispatcher().createServerConnection( + std::move(socket), std::move(transport_socket), *stream_info); + if (const auto timeout = filter_chain->transportSocketConnectTimeout(); + timeout != std::chrono::milliseconds::zero()) { + server_conn_ptr->setTransportSocketConnectTimeout(timeout); + } + server_conn_ptr->setBufferLimits(config_->perConnectionBufferLimitBytes()); + RELEASE_ASSERT(server_conn_ptr->addressProvider().remoteAddress() != nullptr, ""); + const bool empty_filter_chain = !config_->filterChainFactory().createNetworkFilterChain( + *server_conn_ptr, filter_chain->networkFilterFactories()); + if (empty_filter_chain) { + ENVOY_CONN_LOG(debug, "closing connection from {}: no filters", *server_conn_ptr, + server_conn_ptr->addressProvider().remoteAddress()->asString()); + server_conn_ptr->close(Network::ConnectionCloseType::NoFlush); + } + newActiveConnection(*filter_chain, std::move(server_conn_ptr), std::move(stream_info)); +} + +} // namespace Server +} // namespace Envoy \ No newline at end of file diff --git a/source/server/active_stream_listener_base.h b/source/server/active_stream_listener_base.h new file mode 100644 index 0000000000000..39fa660ca4bb1 --- /dev/null +++ b/source/server/active_stream_listener_base.h @@ -0,0 +1,115 @@ +#pragma once + +#include +#include +#include +#include + +#include "envoy/common/time.h" +#include "envoy/event/dispatcher.h" +#include "envoy/network/connection.h" +#include "envoy/network/connection_handler.h" +#include "envoy/network/listener.h" +#include "envoy/stream_info/stream_info.h" + +#include "source/common/common/linked_object.h" +#include "source/server/active_listener_base.h" +#include "source/server/active_tcp_socket.h" + +namespace Envoy { +namespace Server { + +// The base class of the stream listener. It owns the the active sockets that drive itself through +// the listener filters. After the active socket passes all the listener filters, a server +// connection is created. The derived listener must override ``newActiveConnection`` to take the +// ownership of that server connection. +class ActiveStreamListenerBase : public ActiveListenerImplBase, + protected Logger::Loggable { +public: + ActiveStreamListenerBase(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, + Network::ListenerPtr&& listener, Network::ListenerConfig& config); + static void emitLogs(Network::ListenerConfig& config, StreamInfo::StreamInfo& stream_info); + + Event::Dispatcher& dispatcher() { return dispatcher_; } + + /** + * Schedule to remove and destroy the active connections which are not tracked by listener + * config. Caution: The connection are not destroyed yet when function returns. + */ + void + deferredRemoveFilterChains(const std::list& draining_filter_chains) { + // Need to recover the original deleting state. + const bool was_deleting = is_deleting_; + is_deleting_ = true; + for (const auto* filter_chain : draining_filter_chains) { + deferRemoveFilterChain(filter_chain); + } + is_deleting_ = was_deleting; + } + + virtual void incNumConnections() PURE; + virtual void decNumConnections() PURE; + + /** + * Create a new connection from a socket accepted by the listener. + */ + void newConnection(Network::ConnectionSocketPtr&& socket, + std::unique_ptr stream_info); + /** + * Schedule to remove and destroy the active connections owned by the filter chain. + */ + virtual void deferRemoveFilterChain(const Network::FilterChain* filter_chain) PURE; + + virtual Network::BalancedConnectionHandlerOptRef + getBalancedHandlerByAddress(const Network::Address::Instance& address) PURE; + + void onSocketAccepted(std::unique_ptr active_socket) { + // Create and run the filters + config_->filterChainFactory().createListenerFilterChain(*active_socket); + active_socket->continueFilterChain(true); + + // Move active_socket to the sockets_ list if filter iteration needs to continue later. + // Otherwise we let active_socket be destructed when it goes out of scope. + if (active_socket->iter_ != active_socket->accept_filters_.end()) { + active_socket->startTimer(); + LinkedList::moveIntoListBack(std::move(active_socket), sockets_); + } else { + if (!active_socket->connected_) { + // If active_socket is about to be destructed, emit logs if a connection is not created. + if (active_socket->stream_info_ != nullptr) { + emitLogs(*config_, *active_socket->stream_info_); + } else { + // If the active_socket is not connected, this socket is not promoted to active + // connection. Thus the stream_info_ is owned by this active socket. + ENVOY_BUG(active_socket->stream_info_ != nullptr, + "the unconnected active socket must have stream info."); + } + } + } + } + + // Below members are open to access by ActiveTcpSocket. + Network::ConnectionHandler& parent_; + const std::chrono::milliseconds listener_filters_timeout_; + const bool continue_on_listener_filters_timeout_; + std::list> sockets_; + +protected: + /** + * Create the active connection from server connection. This active listener owns the created + * active connection. + * + * @param filter_chain The network filter chain linking to the connection. + * @param server_conn_ptr The server connection. + * @param stream_info The stream info of the active connection. + */ + virtual void newActiveConnection(const Network::FilterChain& filter_chain, + Network::ServerConnectionPtr server_conn_ptr, + std::unique_ptr stream_info) PURE; + Event::Dispatcher& dispatcher_; + Network::ListenerPtr listener_; + bool is_deleting_{false}; +}; + +} // namespace Server +} // namespace Envoy From ef4fc26c9227c78c12365bea95928920d4b3987e Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 15 Jul 2021 17:53:20 +0000 Subject: [PATCH 19/23] new line at EOF Signed-off-by: Yuchen Dai --- source/server/active_stream_listener_base.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/server/active_stream_listener_base.cc b/source/server/active_stream_listener_base.cc index 1a3fe28233167..146d2ff13cac7 100644 --- a/source/server/active_stream_listener_base.cc +++ b/source/server/active_stream_listener_base.cc @@ -59,4 +59,4 @@ void ActiveStreamListenerBase::newConnection(Network::ConnectionSocketPtr&& sock } } // namespace Server -} // namespace Envoy \ No newline at end of file +} // namespace Envoy From feebbc3062b9e898eb6dfcbd1569f9a010982bec Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 21 Jul 2021 22:30:39 +0000 Subject: [PATCH 20/23] address comments Signed-off-by: Yuchen Dai --- source/server/active_stream_listener_base.cc | 2 +- source/server/active_stream_listener_base.h | 21 ++++++++++++-------- source/server/active_tcp_listener.cc | 11 +++++----- source/server/active_tcp_listener.h | 8 +------- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/source/server/active_stream_listener_base.cc b/source/server/active_stream_listener_base.cc index 146d2ff13cac7..39d336034c5fc 100644 --- a/source/server/active_stream_listener_base.cc +++ b/source/server/active_stream_listener_base.cc @@ -12,7 +12,7 @@ ActiveStreamListenerBase::ActiveStreamListenerBase(Network::ConnectionHandler& p : ActiveListenerImplBase(parent, &config), parent_(parent), listener_filters_timeout_(config.listenerFiltersTimeout()), continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()), - dispatcher_(dispatcher), listener_(std::move(listener)) {} + listener_(std::move(listener)), dispatcher_(dispatcher) {} void ActiveStreamListenerBase::emitLogs(Network::ListenerConfig& config, StreamInfo::StreamInfo& stream_info) { diff --git a/source/server/active_stream_listener_base.h b/source/server/active_stream_listener_base.h index 39fa660ca4bb1..63947989e7a8d 100644 --- a/source/server/active_stream_listener_base.h +++ b/source/server/active_stream_listener_base.h @@ -19,10 +19,10 @@ namespace Envoy { namespace Server { -// The base class of the stream listener. It owns the the active sockets that drive itself through -// the listener filters. After the active socket passes all the listener filters, a server -// connection is created. The derived listener must override ``newActiveConnection`` to take the -// ownership of that server connection. +// The base class of the stream listener. It owns listener filter handling of active sockets. +// After the active socket passes all the listener filters, a server connection is created. The +// derived listener must override ``newActiveConnection`` to take the ownership of that server +// connection. class ActiveStreamListenerBase : public ActiveListenerImplBase, protected Logger::Loggable { public: @@ -42,7 +42,7 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, const bool was_deleting = is_deleting_; is_deleting_ = true; for (const auto* filter_chain : draining_filter_chains) { - deferRemoveFilterChain(filter_chain); + removeFilterChain(filter_chain); } is_deleting_ = was_deleting; } @@ -56,9 +56,9 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, void newConnection(Network::ConnectionSocketPtr&& socket, std::unique_ptr stream_info); /** - * Schedule to remove and destroy the active connections owned by the filter chain. + * Schedule removal and destruction of all active connections owned by a filter chain. */ - virtual void deferRemoveFilterChain(const Network::FilterChain* filter_chain) PURE; + virtual void removeFilterChain(const Network::FilterChain* filter_chain) PURE; virtual Network::BalancedConnectionHandlerOptRef getBalancedHandlerByAddress(const Network::Address::Instance& address) PURE; @@ -106,9 +106,14 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, virtual void newActiveConnection(const Network::FilterChain& filter_chain, Network::ServerConnectionPtr server_conn_ptr, std::unique_ptr stream_info) PURE; - Event::Dispatcher& dispatcher_; Network::ListenerPtr listener_; + // True if the follow up connection deletion is raised by the connection collection deletion is performing. + // Otherwise, the collection should be deleted when the last connection in the collection is removed. + // This state is maintained in base class because this state is independent from concrete connection type. bool is_deleting_{false}; + +private: + Event::Dispatcher& dispatcher_; }; } // namespace Server diff --git a/source/server/active_tcp_listener.cc b/source/server/active_tcp_listener.cc index c843f8aed8c53..fe22b5d9aa1f6 100644 --- a/source/server/active_tcp_listener.cc +++ b/source/server/active_tcp_listener.cc @@ -33,9 +33,8 @@ ActiveTcpListener::ActiveTcpListener(Network::TcpConnectionHandler& parent, } ActiveTcpListener::~ActiveTcpListener() { - config_->connectionBalancer().unregisterHandler(*this); - is_deleting_ = true; + config_->connectionBalancer().unregisterHandler(*this); // Purge sockets that have not progressed to connections. This should only happen when // a listener filter stops iteration and never resumes. @@ -44,9 +43,9 @@ ActiveTcpListener::~ActiveTcpListener() { dispatcher().deferredDelete(std::move(removed)); } - for (auto& chain_and_connections : connections_by_context_) { - ASSERT(chain_and_connections.second != nullptr); - auto& connections = chain_and_connections.second->connections_; + for (auto& [chain, active_connections] : connections_by_context_) { + ASSERT(active_connections != nullptr); + auto& connections = active_connections->connections_; while (!connections.empty()) { connections.front()->connection_->close(Network::ConnectionCloseType::NoFlush); } @@ -88,7 +87,7 @@ void ActiveTcpListener::updateListenerConfig(Network::ListenerConfig& config) { config_ = &config; } -void ActiveTcpListener::deferRemoveFilterChain(const Network::FilterChain* filter_chain) { +void ActiveTcpListener::removeFilterChain(const Network::FilterChain* filter_chain) { auto iter = connections_by_context_.find(filter_chain); if (iter == connections_by_context_.end()) { // It is possible when listener is stopping. diff --git a/source/server/active_tcp_listener.h b/source/server/active_tcp_listener.h index 67d4049951456..20f119fbd761c 100644 --- a/source/server/active_tcp_listener.h +++ b/source/server/active_tcp_listener.h @@ -74,9 +74,6 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, void onAcceptWorker(Network::ConnectionSocketPtr&& socket, bool hand_off_restored_destination_connections, bool rebalanced) override; - /** - * Create active connection from the server connection. - */ void newActiveConnection(const Network::FilterChain& filter_chain, Network::ServerConnectionPtr server_conn_ptr, std::unique_ptr stream_info) override; @@ -92,10 +89,7 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, */ void updateListenerConfig(Network::ListenerConfig& config); - /** - * Schedule to remove and destroy the active connection owned by the filter chain. - */ - void deferRemoveFilterChain(const Network::FilterChain* filter_chain) override; + void removeFilterChain(const Network::FilterChain* filter_chain) override; /** * Remove and destroy an active connection. From df6f495172e1123b42d47e685f27dbacb4e23bcb Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 22 Jul 2021 00:23:24 +0000 Subject: [PATCH 21/23] format Signed-off-by: Yuchen Dai --- source/server/active_stream_listener_base.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/server/active_stream_listener_base.h b/source/server/active_stream_listener_base.h index 63947989e7a8d..d0fbd485cc97d 100644 --- a/source/server/active_stream_listener_base.h +++ b/source/server/active_stream_listener_base.h @@ -107,9 +107,10 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, Network::ServerConnectionPtr server_conn_ptr, std::unique_ptr stream_info) PURE; Network::ListenerPtr listener_; - // True if the follow up connection deletion is raised by the connection collection deletion is performing. - // Otherwise, the collection should be deleted when the last connection in the collection is removed. - // This state is maintained in base class because this state is independent from concrete connection type. + // True if the follow up connection deletion is raised by the connection collection deletion is + // performing. Otherwise, the collection should be deleted when the last connection in the + // collection is removed. This state is maintained in base class because this state is independent + // from concrete connection type. bool is_deleting_{false}; private: From fa60a0963b1ff9e84a14bd95fc2e18f993c393a1 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 22 Jul 2021 17:54:04 +0000 Subject: [PATCH 22/23] revert flat_hash_map Signed-off-by: Yuchen Dai --- source/server/active_tcp_listener.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/server/active_tcp_listener.h b/source/server/active_tcp_listener.h index 95eeeef2ff8c2..9ea378f445395 100644 --- a/source/server/active_tcp_listener.h +++ b/source/server/active_tcp_listener.h @@ -9,8 +9,6 @@ #include "source/server/active_stream_listener_base.h" #include "source/server/active_tcp_socket.h" -#include "absl/container/node_hash_map.h" - namespace Envoy { namespace Server { @@ -98,7 +96,7 @@ class ActiveTcpListener final : public Network::TcpListenerCallbacks, */ void removeConnection(ActiveTcpConnection& connection); - absl::node_hash_map> + absl::flat_hash_map> connections_by_context_; Network::TcpConnectionHandler& tcp_conn_handler_; From 87060fcfa863a1a27c942b3cb63224d8a32ecac6 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 22 Jul 2021 19:25:01 +0000 Subject: [PATCH 23/23] base listener: add removeSocket and const socket list accessor Signed-off-by: Yuchen Dai --- source/server/active_stream_listener_base.h | 20 +++++++++++++++++++- source/server/active_tcp_socket.cc | 2 +- test/server/active_tcp_listener_test.cc | 4 ++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/source/server/active_stream_listener_base.h b/source/server/active_stream_listener_base.h index d0fbd485cc97d..2d99e3965bf69 100644 --- a/source/server/active_stream_listener_base.h +++ b/source/server/active_stream_listener_base.h @@ -55,6 +55,23 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, */ void newConnection(Network::ConnectionSocketPtr&& socket, std::unique_ptr stream_info); + + /** + * Remove the socket from this listener. Should be called when the socket passes the listener + * filter. + * @return std::unique_ptr the exact same socket in the parameter but in the + * state that not owned by the listener. + */ + std::unique_ptr removeSocket(ActiveTcpSocket&& socket) { + return socket.removeFromList(sockets_); + } + + /** + * @return const std::list>& the sockets going through the + * listener filters. + */ + const std::list>& sockets() const { return sockets_; } + /** * Schedule removal and destruction of all active connections owned by a filter chain. */ @@ -92,7 +109,6 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, Network::ConnectionHandler& parent_; const std::chrono::milliseconds listener_filters_timeout_; const bool continue_on_listener_filters_timeout_; - std::list> sockets_; protected: /** @@ -106,6 +122,8 @@ class ActiveStreamListenerBase : public ActiveListenerImplBase, virtual void newActiveConnection(const Network::FilterChain& filter_chain, Network::ServerConnectionPtr server_conn_ptr, std::unique_ptr stream_info) PURE; + + std::list> sockets_; Network::ListenerPtr listener_; // True if the follow up connection deletion is raised by the connection collection deletion is // performing. Otherwise, the collection should be deleted when the last connection in the diff --git a/source/server/active_tcp_socket.cc b/source/server/active_tcp_socket.cc index 33740b957a42d..e939bffc26128 100644 --- a/source/server/active_tcp_socket.cc +++ b/source/server/active_tcp_socket.cc @@ -59,7 +59,7 @@ void ActiveTcpSocket::startTimer() { } void ActiveTcpSocket::unlink() { - auto removed = removeFromList(listener_.sockets_); + auto removed = listener_.removeSocket(std::move(*this)); if (removed->timer_ != nullptr) { removed->timer_->disableTimer(); } diff --git a/test/server/active_tcp_listener_test.cc b/test/server/active_tcp_listener_test.cc index baaf57421ec08..8284559cd5ca8 100644 --- a/test/server/active_tcp_listener_test.cc +++ b/test/server/active_tcp_listener_test.cc @@ -110,9 +110,9 @@ TEST_F(ActiveTcpListenerTest, PopulateSNIWhenActiveTcpSocketTimeout) { // calling the onAcceptWorker() to create the ActiveTcpSocket. active_listener->onAcceptWorker(std::move(accepted_socket), false, false); // get the ActiveTcpSocket pointer before unlink() removed from the link-list. - ActiveTcpSocket* tcp_socket = active_listener->sockets_.front().get(); + ActiveTcpSocket* tcp_socket = active_listener->sockets().front().get(); // trigger the onTimeout event manually, since the timer is fake. - active_listener->sockets_.front()->onTimeout(); + active_listener->sockets().front()->onTimeout(); EXPECT_EQ(server_name, tcp_socket->stream_info_->downstreamAddressProvider().requestedServerName());