From c00cc79e5039cd4cf1876e24fba390bfdc5aef42 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 2 Sep 2021 17:46:52 +0000 Subject: [PATCH 01/59] wip Signed-off-by: Yuchen Dai --- envoy/network/BUILD | 8 ++++++ envoy/network/client_connection_manager.cc | 0 envoy/network/client_connection_manager.h | 19 +++++++++++++ source/common/event/BUILD | 2 ++ source/common/event/dispatcher_impl.cc | 26 +++++++++++++++-- source/extensions/io_socket/user_space/BUILD | 16 +++++++++++ .../user_space/client_connection_factory.cc | 16 +++++++++++ .../user_space/client_connection_factory.h | 28 +++++++++++++++++++ 8 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 envoy/network/client_connection_manager.cc create mode 100644 envoy/network/client_connection_manager.h create mode 100644 source/extensions/io_socket/user_space/client_connection_factory.cc create mode 100644 source/extensions/io_socket/user_space/client_connection_factory.h diff --git a/envoy/network/BUILD b/envoy/network/BUILD index 3caab27a2aaa9..3ed9fa0ae395d 100644 --- a/envoy/network/BUILD +++ b/envoy/network/BUILD @@ -31,6 +31,14 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "client_connection_manager", + hdrs = ["client_connection_manager.h"], + deps = [ + ":connection_interface", + ], +) + envoy_cc_library( name = "connection_handler_interface", hdrs = ["connection_handler.h"], diff --git a/envoy/network/client_connection_manager.cc b/envoy/network/client_connection_manager.cc new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/envoy/network/client_connection_manager.h b/envoy/network/client_connection_manager.h new file mode 100644 index 0000000000000..15c465585629e --- /dev/null +++ b/envoy/network/client_connection_manager.h @@ -0,0 +1,19 @@ +#pragma once + +#include "envoy/network/connection.h" + +namespace Envoy { +namespace Network { + +class CCContext {}; +class ClientConnectionFactory { +public: + virtual ~ClientConnectionFactory() = default; + std::string category() { return "network.connection"; } + virtual std::string name() PURE; + + virtual Network::ClientConnectionPtr createClientConnection(const CCContext& context) PURE; +}; + +} // namespace Network +} // namespace Envoy diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 11ba35646e9af..5931a842a4b94 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -31,6 +31,8 @@ envoy_cc_library( "//conditions:default": "posix", }), deps = [ + "//envoy/network:client_connection_manager", + "//source/common/config:utility_lib", ":dispatcher_includes", ":libevent_scheduler_lib", ":real_time_system_lib", diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index a7a30838aee9c..e6e2f31607390 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -28,6 +28,9 @@ #include "source/common/network/udp_listener_impl.h" #include "source/common/runtime/runtime_features.h" +#include "source/common/config/utility.h" +#include "envoy/network/client_connection_manager.h" + #include "event2/event.h" #ifdef ENVOY_HANDLE_SIGNALS @@ -153,8 +156,27 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) { ASSERT(isThreadSafe()); - return std::make_unique(*this, address, source_address, - std::move(transport_socket), options); + + auto address_type_name = [](const Network::Address::InstanceConstSharedPtr& addr) { + ASSERT(addr != nullptr); + switch (addr->type()) { + // TODO: create IP and + case Network::Address::Type::Ip: + case Network::Address::Type::Pipe: + return "does-not-exist"; + case Network::Address::Type::EnvoyInternal: + return "envoy-internal"; + } + }; + // TODO: register and find by address type instead of name. + auto factory = Config::Utility::getFactoryByName( + address_type_name(address)); + if (factory == nullptr) { + // get rid of this once the ip and pipe factory is offered. + return std::make_unique(*this, address, source_address, + std::move(transport_socket), options); + } + return factory->createClientConnection(Network::CCContext()); } Network::DnsResolverSharedPtr DispatcherImpl::createDnsResolver( diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 01a5948f4c9e4..ace687cebbc48 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -13,6 +13,7 @@ envoy_cc_extension( name = "config", srcs = ["config.h"], deps = [ + ":client_connection_factory", ":io_handle_impl_lib", ], ) @@ -57,3 +58,18 @@ envoy_cc_library( "//source/common/network:default_socket_interface_lib", ], ) + +envoy_cc_library( + name = "client_connection_factory", + srcs = [ + "client_connection_factory.cc", + ], + hdrs = [ + "client_connection_factory.h", + ], + deps = [ + "//envoy/network:client_connection_manager", + "//envoy/network:connection_interface", + "//envoy/registry", + ], +) diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc new file mode 100644 index 0000000000000..7952e0b227889 --- /dev/null +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -0,0 +1,16 @@ +#include "client_connection_factory.h" + +#include "envoy/registry/registry.h" +//#include "envoy/singleton/manager.h" + +namespace Envoy { + +namespace Extensions { +namespace IoSocket { +namespace UserSpace { + +REGISTER_FACTORY(InternalClientConnectionFactory, Network::ClientConnectionFactory); +} +} // namespace IoSocket +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/io_socket/user_space/client_connection_factory.h new file mode 100644 index 0000000000000..628660a58f181 --- /dev/null +++ b/source/extensions/io_socket/user_space/client_connection_factory.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +#include "envoy/common/pure.h" + +#include "envoy/network/connection.h" +#include "envoy/network/client_connection_manager.h" + +namespace Envoy { + +namespace Extensions { +namespace IoSocket { +namespace UserSpace { + +class InternalClientConnectionFactory : public Network::ClientConnectionFactory { +public: + std::string name() override { return "EnvoyInternal"; } + Network::ClientConnectionPtr createClientConnection(const Network::CCContext&) override { + return nullptr; + } +}; + +} // namespace UserSpace +} // namespace IoSocket +} // namespace Extensions +} // namespace Envoy From e037f8e5408704dd8c8490e6f2e5c52eb54fc494 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 9 Sep 2021 16:16:48 +0000 Subject: [PATCH 02/59] stash client conn fac Signed-off-by: Yuchen Dai --- source/extensions/io_socket/user_space/BUILD | 3 + .../user_space/client_connection_factory.cc | 62 ++++++++++++++++++- .../user_space/client_connection_factory.h | 4 +- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index ace687cebbc48..1d4a144362202 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -68,8 +68,11 @@ envoy_cc_library( "client_connection_factory.h", ], deps = [ + ":io_handle_impl_lib", "//envoy/network:client_connection_manager", "//envoy/network:connection_interface", "//envoy/registry", + "//source/common/network:connection_lib", + "//source/common/network:listen_socket_lib", ], ) diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index 7952e0b227889..4f7e88b793e6d 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -1,7 +1,11 @@ -#include "client_connection_factory.h" +#include "source/extensions/io_socket/user_space/client_connection_factory.h" #include "envoy/registry/registry.h" -//#include "envoy/singleton/manager.h" + +#include "source/extensions/io_socket/user_space/io_handle_impl.h" +#include "source/common/network/connection_impl.h" +#include "source/common/network/address_impl.h" +#include "source/common/network/listen_socket_impl.h" namespace Envoy { @@ -9,8 +13,60 @@ namespace Extensions { namespace IoSocket { namespace UserSpace { -REGISTER_FACTORY(InternalClientConnectionFactory, Network::ClientConnectionFactory); +Network::ClientConnectionPtr +InternalClientConnectionFactory::createClientConnection(const Network::CCContext&) { + // TODO: fetch from arg + + Network::Address::InstanceConstSharedPtr address; + Network::Address::InstanceConstSharedPtr source_address; + Network::TransportSocketPtr transport_socket; + const Network::ConnectionSocket::OptionsSharedPtr options; + Event::Dispatcher& dispatcher = *reinterpret_cast(1); + + Network::IoHandlePtr io_handle_client; + Network::IoHandlePtr io_handle_server; + + std::tie(io_handle_client, io_handle_server) = + Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); + + auto client_conn = std::make_unique( + *this, std::move(io_handle_client), address, source_address, std::move(transport_socket), + options); + // TODO(lambdai): refactor nested if. + auto internal_listener_manager = dispatcher.getInternalListenerManager(); + if (internal_listener_manager.has_value()) { + // It's either in main thread or the worker is not yet started. + auto internal_listener = internal_listener_manager.value().get().findByAddress(address); + if (internal_listener.has_value()) { + auto original_address = address; + // if (options != nullptr) { + // for (const auto& opt : *options) { + // auto* internal_opt = dynamic_cast(opt.get()); + // if (internal_opt != nullptr) { + // original_address = internal_opt->original_remote_address_; + // } + // } + // } + auto accepted_socket = std::make_unique( + std::move(io_handle_server), original_address, source_address); + // TODO: also check if disabled + internal_listener.value().get().onAccept(std::move(accepted_socket)); + FANCY_LOG(debug, "lambdai: find internal listener {} ", address->asStringView()); + } else { + FANCY_LOG(debug, "lambdai: cannot find internal listener {} ", address->asStringView()); + // injected error into client_conn; + io_handle_server->close(); + } + return client_conn; + } else { + // TODO(lambdai): return closed connection. + return nullptr; + } } +REGISTER_FACTORY(InternalClientConnectionFactory, Network::ClientConnectionFactory); + +} // namespace UserSpace } // namespace IoSocket } // namespace Extensions } // namespace Envoy +} // namespace Envoy diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/io_socket/user_space/client_connection_factory.h index 628660a58f181..8afbe766e37c6 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.h +++ b/source/extensions/io_socket/user_space/client_connection_factory.h @@ -17,9 +17,7 @@ namespace UserSpace { class InternalClientConnectionFactory : public Network::ClientConnectionFactory { public: std::string name() override { return "EnvoyInternal"; } - Network::ClientConnectionPtr createClientConnection(const Network::CCContext&) override { - return nullptr; - } + Network::ClientConnectionPtr createClientConnection(const Network::CCContext&) override; }; } // namespace UserSpace From 9ede26a82d85157428ee1682e6472cd39ec4de98 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 10 Sep 2021 21:33:49 +0000 Subject: [PATCH 03/59] make it build Signed-off-by: Yuchen Dai --- envoy/event/dispatcher.h | 8 ++++ envoy/network/client_connection_manager.cc | 1 + envoy/network/listener.h | 44 +++++++++++++++++++ source/common/event/dispatcher_impl.cc | 5 +-- source/common/event/dispatcher_impl.h | 12 +++++ .../user_space/client_connection_factory.cc | 18 ++++---- .../user_space/client_connection_factory.h | 3 +- source/server/admin/admin.h | 3 ++ source/server/listener_impl.h | 3 ++ 9 files changed, 84 insertions(+), 13 deletions(-) diff --git a/envoy/event/dispatcher.h b/envoy/event/dispatcher.h index 9b887a0c4b268..95af640b4e7a1 100644 --- a/envoy/event/dispatcher.h +++ b/envoy/event/dispatcher.h @@ -218,6 +218,14 @@ class Dispatcher : public DispatcherBase, public ScopeTracker { Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) PURE; + /** + * Register an internal listener manager for this dispatcher. + */ + virtual void + registerInternalListenerManager(Network::InternalListenerManager& internal_listener_manager) PURE; + + virtual Network::InternalListenerManagerOptRef getInternalListenerManager() PURE; + /** * Creates an async DNS resolver. The resolver should only be used on the thread that runs this * dispatcher. diff --git a/envoy/network/client_connection_manager.cc b/envoy/network/client_connection_manager.cc index e69de29bb2d1d..fafc4c928ff93 100644 --- a/envoy/network/client_connection_manager.cc +++ b/envoy/network/client_connection_manager.cc @@ -0,0 +1 @@ +// NOLINT(namespace-envoy) diff --git a/envoy/network/listener.h b/envoy/network/listener.h index ad2b69a6c7502..4f9d8b0c1a646 100644 --- a/envoy/network/listener.h +++ b/envoy/network/listener.h @@ -106,6 +106,16 @@ class UdpListenerConfig { using UdpListenerConfigOptRef = OptRef; +/** + * Configuration for an internal listener. + */ +class InternalListenerConfig { +public: + virtual ~InternalListenerConfig() = default; +}; + +using InternalListenerConfigOptRef = OptRef; + /** * A configuration for an individual listener. */ @@ -184,6 +194,11 @@ class ListenerConfig { */ virtual UdpListenerConfigOptRef udpListenerConfig() PURE; + /** + * @return the internal configuration for the listener IFF it is an internal listener. + */ + virtual InternalListenerConfigOptRef internalListenerConfig() PURE; + /** * @return traffic direction of the listener. */ @@ -426,6 +441,35 @@ class UdpListener : public virtual Listener { }; using UdpListenerPtr = std::unique_ptr; +class InternalListenerCallbacks { +public: + virtual ~InternalListenerCallbacks() = default; + + /** + * Called when a new connection is accepted. + * @param socket supplies the socket that is moved into the callee. + */ + virtual void onAccept(ConnectionSocketPtr&& socket) PURE; + + virtual Event::Dispatcher& dispatcher() PURE; +}; +using InternalListenerCallbacksOptRef = + absl::optional>; + +class InternalListener {}; + +using InternalListenerPtr = std::unique_ptr; +using InternalListenerOptRef = absl::optional>; + +class InternalListenerManager { +public: + virtual ~InternalListenerManager() = default; + virtual InternalListenerCallbacksOptRef + findByAddress(const Address::InstanceConstSharedPtr& listen_address) PURE; +}; + +using InternalListenerManagerOptRef = + absl::optional>; /** * Handles delivering datagrams to the correct worker. diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index e6e2f31607390..9a04f74c960ab 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -9,6 +9,7 @@ #include "envoy/api/api.h" #include "envoy/common/scope_tracker.h" #include "envoy/config/overload/v3/overload.pb.h" +#include "envoy/network/client_connection_manager.h" #include "envoy/network/listen_socket.h" #include "envoy/network/listener.h" @@ -16,6 +17,7 @@ #include "source/common/common/assert.h" #include "source/common/common/lock_guard.h" #include "source/common/common/thread.h" +#include "source/common/config/utility.h" #include "source/common/event/file_event_impl.h" #include "source/common/event/libevent_scheduler.h" #include "source/common/event/scaled_range_timer_manager_impl.h" @@ -28,9 +30,6 @@ #include "source/common/network/udp_listener_impl.h" #include "source/common/runtime/runtime_features.h" -#include "source/common/config/utility.h" -#include "envoy/network/client_connection_manager.h" - #include "event2/event.h" #ifdef ENVOY_HANDLE_SIGNALS diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index d98b5267c1e5c..1faad6cf3e1d2 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -66,6 +66,17 @@ class DispatcherImpl : Logger::Loggable, Network::Address::InstanceConstSharedPtr source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) override; + + void registerInternalListenerManager( + Network::InternalListenerManager& internal_listener_manager) override { + ASSERT(!internal_listener_manager_.has_value()); + internal_listener_manager_ = internal_listener_manager; + } + + Network::InternalListenerManagerOptRef getInternalListenerManager() override { + return internal_listener_manager_; + } + Network::DnsResolverSharedPtr createDnsResolver( const std::vector& resolvers, const envoy::config::core::v3::DnsResolverOptions& dns_resolver_options) override; @@ -175,6 +186,7 @@ class DispatcherImpl : Logger::Loggable, MonotonicTime approximate_monotonic_time_; WatchdogRegistrationPtr watchdog_registration_; const ScaledRangeTimerManagerPtr scaled_timer_manager_; + Network::InternalListenerManagerOptRef internal_listener_manager_; }; } // namespace Event diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index 4f7e88b793e6d..171f603d1fc4c 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -2,10 +2,10 @@ #include "envoy/registry/registry.h" -#include "source/extensions/io_socket/user_space/io_handle_impl.h" -#include "source/common/network/connection_impl.h" #include "source/common/network/address_impl.h" +#include "source/common/network/connection_impl.h" #include "source/common/network/listen_socket_impl.h" +#include "source/extensions/io_socket/user_space/io_handle_impl.h" namespace Envoy { @@ -30,8 +30,10 @@ InternalClientConnectionFactory::createClientConnection(const Network::CCContext Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); auto client_conn = std::make_unique( - *this, std::move(io_handle_client), address, source_address, std::move(transport_socket), - options); + dispatcher, + std::make_unique(std::move(io_handle_client), source_address, + address), + source_address, std::move(transport_socket), options); // TODO(lambdai): refactor nested if. auto internal_listener_manager = dispatcher.getInternalListenerManager(); if (internal_listener_manager.has_value()) { @@ -57,11 +59,12 @@ InternalClientConnectionFactory::createClientConnection(const Network::CCContext // injected error into client_conn; io_handle_server->close(); } - return client_conn; } else { - // TODO(lambdai): return closed connection. - return nullptr; + FANCY_LOG(debug, "lambdai: cannot find internal listener {} ", address->asStringView()); + // injected error into client_conn; + io_handle_server->close(); } + return client_conn; } REGISTER_FACTORY(InternalClientConnectionFactory, Network::ClientConnectionFactory); @@ -69,4 +72,3 @@ REGISTER_FACTORY(InternalClientConnectionFactory, Network::ClientConnectionFacto } // namespace IoSocket } // namespace Extensions } // namespace Envoy -} // namespace Envoy diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/io_socket/user_space/client_connection_factory.h index 8afbe766e37c6..8ed9873b22779 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.h +++ b/source/extensions/io_socket/user_space/client_connection_factory.h @@ -4,9 +4,8 @@ #include #include "envoy/common/pure.h" - -#include "envoy/network/connection.h" #include "envoy/network/client_connection_manager.h" +#include "envoy/network/connection.h" namespace Envoy { diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index e99330c2c8fc7..4219418426c38 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -360,6 +360,9 @@ class AdminImpl : public Admin, Network::UdpListenerConfigOptRef udpListenerConfig() override { return Network::UdpListenerConfigOptRef(); } + Network::InternalListenerConfigOptRef internalListenerConfig() override { + return Network::InternalListenerConfigOptRef(); + } envoy::config::core::v3::TrafficDirection direction() const override { return envoy::config::core::v3::UNSPECIFIED; } diff --git a/source/server/listener_impl.h b/source/server/listener_impl.h index 0114ff9c9e33e..c7ff942457774 100644 --- a/source/server/listener_impl.h +++ b/source/server/listener_impl.h @@ -313,6 +313,9 @@ class ListenerImpl final : public Network::ListenerConfig, return udp_listener_config_ != nullptr ? *udp_listener_config_ : Network::UdpListenerConfigOptRef(); } + Network::InternalListenerConfigOptRef internalListenerConfig() override { + return Network::InternalListenerConfigOptRef(); + } Network::ConnectionBalancer& connectionBalancer() override { return *connection_balancer_; } ResourceLimit& openConnections() override { return *open_connections_; } const std::vector& accessLogs() const override { From 14ae9dc59cb7860e7d03ccdbb5fd0daaa9977592 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 10 Sep 2021 23:40:50 +0000 Subject: [PATCH 04/59] format Signed-off-by: Yuchen Dai --- envoy/network/BUILD | 3 +++ envoy/network/client_connection_manager.h | 11 +++++++++-- source/common/event/dispatcher_impl.cc | 3 ++- .../user_space/client_connection_factory.cc | 14 +++++--------- .../user_space/client_connection_factory.h | 7 ++++++- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/envoy/network/BUILD b/envoy/network/BUILD index 3ed9fa0ae395d..e1af15a402f56 100644 --- a/envoy/network/BUILD +++ b/envoy/network/BUILD @@ -35,7 +35,10 @@ envoy_cc_library( name = "client_connection_manager", hdrs = ["client_connection_manager.h"], deps = [ + ":address_interface", ":connection_interface", + ":listen_socket_interface", + ":transport_socket_interface", ], ) diff --git a/envoy/network/client_connection_manager.h b/envoy/network/client_connection_manager.h index 15c465585629e..8375dc8ae2583 100644 --- a/envoy/network/client_connection_manager.h +++ b/envoy/network/client_connection_manager.h @@ -1,18 +1,25 @@ #pragma once +#include "envoy/network/address.h" #include "envoy/network/connection.h" +#include "envoy/network/listen_socket.h" +#include "envoy/network/transport_socket.h" namespace Envoy { namespace Network { -class CCContext {}; class ClientConnectionFactory { public: virtual ~ClientConnectionFactory() = default; std::string category() { return "network.connection"; } virtual std::string name() PURE; - virtual Network::ClientConnectionPtr createClientConnection(const CCContext& context) PURE; + virtual Network::ClientConnectionPtr + createClientConnection(Event::Dispatcher& dispatcher, + Network::Address::InstanceConstSharedPtr address, + Network::Address::InstanceConstSharedPtr source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) PURE; }; } // namespace Network diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 9a04f74c960ab..2aef75c02a003 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -175,7 +175,8 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr return std::make_unique(*this, address, source_address, std::move(transport_socket), options); } - return factory->createClientConnection(Network::CCContext()); + return factory->createClientConnection(*this, address, source_address, + std::move(transport_socket), options); } Network::DnsResolverSharedPtr DispatcherImpl::createDnsResolver( diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index 171f603d1fc4c..91f211376dc97 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -13,15 +13,11 @@ namespace Extensions { namespace IoSocket { namespace UserSpace { -Network::ClientConnectionPtr -InternalClientConnectionFactory::createClientConnection(const Network::CCContext&) { - // TODO: fetch from arg - - Network::Address::InstanceConstSharedPtr address; - Network::Address::InstanceConstSharedPtr source_address; - Network::TransportSocketPtr transport_socket; - const Network::ConnectionSocket::OptionsSharedPtr options; - Event::Dispatcher& dispatcher = *reinterpret_cast(1); +Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnection( + Event::Dispatcher& dispatcher, Network::Address::InstanceConstSharedPtr address, + Network::Address::InstanceConstSharedPtr source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) { Network::IoHandlePtr io_handle_client; Network::IoHandlePtr io_handle_server; diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/io_socket/user_space/client_connection_factory.h index 8ed9873b22779..5ea485178a9eb 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.h +++ b/source/extensions/io_socket/user_space/client_connection_factory.h @@ -16,7 +16,12 @@ namespace UserSpace { class InternalClientConnectionFactory : public Network::ClientConnectionFactory { public: std::string name() override { return "EnvoyInternal"; } - Network::ClientConnectionPtr createClientConnection(const Network::CCContext&) override; + Network::ClientConnectionPtr + createClientConnection(Event::Dispatcher& dispatcher, + Network::Address::InstanceConstSharedPtr address, + Network::Address::InstanceConstSharedPtr source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) override; }; } // namespace UserSpace From 3758571cd8b1a9af9b3062903482b35aafa97b88 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 13 Sep 2021 08:41:07 -0700 Subject: [PATCH 05/59] adding int conn factory test Signed-off-by: Yuchen Dai --- test/extensions/io_socket/user_space/BUILD | 12 +++ .../client_connection_factory_test.cc | 83 +++++++++++++++++++ test/mocks/event/mocks.h | 3 + 3 files changed, 98 insertions(+) create mode 100644 test/extensions/io_socket/user_space/client_connection_factory_test.cc diff --git a/test/extensions/io_socket/user_space/BUILD b/test/extensions/io_socket/user_space/BUILD index 458d6345d36da..717e33be3c10e 100644 --- a/test/extensions/io_socket/user_space/BUILD +++ b/test/extensions/io_socket/user_space/BUILD @@ -58,3 +58,15 @@ envoy_extension_cc_test( "//test/test_common:network_utility_lib", ], ) + +envoy_extension_cc_test( + name = "client_connection_factory_test", + srcs = ["client_connection_factory_test.cc"], + extension_names = ["envoy.io_socket.user_space"], + deps = [ + "//source/common/common:utility_lib", + "//source/common/network:address_lib", + "//source/extensions/io_socket/user_space:io_handle_impl_lib", + "//test/mocks/event:event_mocks", + ], +) diff --git a/test/extensions/io_socket/user_space/client_connection_factory_test.cc b/test/extensions/io_socket/user_space/client_connection_factory_test.cc new file mode 100644 index 0000000000000..31b841effdd96 --- /dev/null +++ b/test/extensions/io_socket/user_space/client_connection_factory_test.cc @@ -0,0 +1,83 @@ +#include "envoy/buffer/buffer.h" +#include "envoy/event/file_event.h" +#include "envoy/network/listen_socket.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/common/common/fancy_logger.h" +#include "source/common/network/address_impl.h" +#include "source/extensions/io_socket/user_space/io_handle_impl.h" +#include "test/test_common/network_utility.h" + +#include "test/mocks/event/mocks.h" + +#include "absl/container/fixed_array.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; + +namespace Envoy { +namespace Extensions { +namespace IoSocket { +namespace UserSpace { +namespace { +class ClientConnectionFactoryTest : public testing::Test { +public: + ClientConnectionFactoryTest() : buf_(1024) { + std::tie(io_handle_, io_handle_peer_) = IoHandleFactory::createIoHandlePair(); + } + + NiceMock dispatcher_; + + // Owned by IoHandleImpl. + NiceMock* schedulable_cb_; + std::unique_ptr io_handle_; + std::unique_ptr io_handle_peer_; + absl::FixedArray buf_; + Network::Address::EnvoyInternalInstance listener_addr{"listener_internal_address"}; +}; + +class MockInternalListenerCallbacks : public Network::InternalListenerCallbacks { +public: + MOCK_METHOD(void, onAccept, (Network::ConnectionSocketPtr &&)); + MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); +}; + +class MockInternalListenerManger : public Network::InternalListenerManagerOptRef { +public: + MOCK_METHOD(Network::InternalListenerCallbacksOptRef, findByAddress, + (const Network::Address::InstanceConstSharedPtr&)); +}; + +TEST_F(ClientConnectionFactoryTest, Basic) { + auto client_conn = dispatcher_.createClientConnection( + std::make_shared(listener_addr), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + client_conn->close(Network::ConnectionCloseType::NoFlush); +} +TEST_F(ClientConnectionFactoryTest, BasicRecv) { + Buffer::OwnedImpl buf_to_write("0123456789"); + io_handle_peer_->write(buf_to_write); + { + auto result = io_handle_->recv(buf_.data(), buf_.size(), 0); + ASSERT_EQ(10, result.return_value_); + ASSERT_EQ("0123456789", absl::string_view(buf_.data(), result.return_value_)); + } + { + auto result = io_handle_->recv(buf_.data(), buf_.size(), 0); + // `EAGAIN`. + EXPECT_FALSE(result.ok()); + EXPECT_EQ(Api::IoError::IoErrorCode::Again, result.err_->getErrorCode()); + } + { + io_handle_->setWriteEnd(); + auto result = io_handle_->recv(buf_.data(), buf_.size(), 0); + EXPECT_TRUE(result.ok()); + } +} + +} // namespace +} // namespace UserSpace +} // namespace IoSocket +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index 26b1559e5ff95..dfa52e0fe324c 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -57,6 +57,9 @@ class MockDispatcher : public Dispatcher { createClientConnection_(address, source_address, transport_socket, options)}; } + MOCK_METHOD(Network::InternalListenerManagerOptRef, getInternalListenerManager, ()); + MOCK_METHOD(void, registerInternalListenerManager, (Network::InternalListenerManager&)); + FileEventPtr createFileEvent(os_fd_t fd, FileReadyCb cb, FileTriggerType trigger, uint32_t events) override { return FileEventPtr{createFileEvent_(fd, cb, trigger, events)}; From ac3234495a549c2e9ddd236a39df76668f67c514 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 13 Sep 2021 18:07:32 +0000 Subject: [PATCH 06/59] basic internal connection factory test Signed-off-by: Yuchen Dai --- source/common/event/dispatcher_impl.cc | 2 +- test/common/event/BUILD | 1 + test/common/event/dispatcher_impl_test.cc | 31 +++++ .../proxy_protocol_regression_test.cc | 3 + .../proxy_protocol/proxy_protocol_test.cc | 6 + test/extensions/io_socket/user_space/BUILD | 1 + .../client_connection_factory_test.cc | 107 +++++++++++++----- test/integration/fake_upstream.h | 3 + test/mocks/event/wrapped_dispatcher.h | 9 ++ test/mocks/network/mocks.h | 1 + test/server/connection_handler_test.cc | 3 + 11 files changed, 140 insertions(+), 27 deletions(-) diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 2aef75c02a003..1c02de66a7432 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -164,7 +164,7 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr case Network::Address::Type::Pipe: return "does-not-exist"; case Network::Address::Type::EnvoyInternal: - return "envoy-internal"; + return "EnvoyInternal"; } }; // TODO: register and find by address type instead of name. diff --git a/test/common/event/BUILD b/test/common/event/BUILD index c42e8e1aff1ea..8b945fe028615 100644 --- a/test/common/event/BUILD +++ b/test/common/event/BUILD @@ -16,6 +16,7 @@ envoy_cc_test( "//source/common/event:deferred_task", "//source/common/event:dispatcher_includes", "//source/common/event:dispatcher_lib", + "//source/common/network:address_lib", "//source/common/stats:isolated_store_lib", "//test/mocks:common_lib", "//test/mocks/server:watch_dog_mocks", diff --git a/test/common/event/dispatcher_impl_test.cc b/test/common/event/dispatcher_impl_test.cc index 7538a8c495c07..83a32175af04e 100644 --- a/test/common/event/dispatcher_impl_test.cc +++ b/test/common/event/dispatcher_impl_test.cc @@ -11,11 +11,13 @@ #include "source/common/event/deferred_task.h" #include "source/common/event/dispatcher_impl.h" #include "source/common/event/timer_impl.h" +#include "source/common/network/address_impl.h" #include "source/common/stats/isolated_store_impl.h" #include "test/mocks/common.h" #include "test/mocks/server/watch_dog.h" #include "test/mocks/stats/mocks.h" +#include "test/test_common/network_utility.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" @@ -1550,6 +1552,35 @@ TEST_F(DispatcherWithWatchdogTest, TouchBeforeFdEvent) { dispatcher_->run(Dispatcher::RunType::NonBlock); } +class DispatcherConnectionTest : public testing::Test { +protected: + DispatcherConnectionTest() + : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")) {} + + Api::ApiPtr api_; + DispatcherPtr dispatcher_; +}; + +TEST_F(DispatcherConnectionTest, CreateTcpConnection) { + + auto client_conn = dispatcher_->createClientConnection( + std::make_shared("127.0.0.1", 10911), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + EXPECT_NE(nullptr, client_conn); + client_conn->close(Network::ConnectionCloseType::NoFlush); +} + +// If the internal connection factory is not linked, envoy will be dead when creating connection to +// internal address. +TEST_F(DispatcherConnectionTest, CreateEnvoyInternalConnectionWhenFactoryNotExist) { + EXPECT_DEATH( + dispatcher_->createClientConnection( + std::make_shared("listener_internal_address"), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), + nullptr), + "not implemented"); +} + } // namespace } // namespace Event } // namespace Envoy diff --git a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc index cb9d97d5593f8..705f80156db01 100644 --- a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc +++ b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc @@ -72,6 +72,9 @@ class ProxyProtocolRegressionTest : public testing::TestWithParamallocateDispatcher("test_thread")), + buf_(1024) { std::tie(io_handle_, io_handle_peer_) = IoHandleFactory::createIoHandlePair(); } - NiceMock dispatcher_; + Api::ApiPtr api_; + Event::DispatcherPtr dispatcher_; // Owned by IoHandleImpl. NiceMock* schedulable_cb_; @@ -43,41 +51,88 @@ class MockInternalListenerCallbacks : public Network::InternalListenerCallbacks MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); }; -class MockInternalListenerManger : public Network::InternalListenerManagerOptRef { +class MockInternalListenerManger : public Network::InternalListenerManager { public: MOCK_METHOD(Network::InternalListenerCallbacksOptRef, findByAddress, (const Network::Address::InstanceConstSharedPtr&)); }; -TEST_F(ClientConnectionFactoryTest, Basic) { - auto client_conn = dispatcher_.createClientConnection( +TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalConnectionManagerNotExist) { + auto client_conn = dispatcher_->createClientConnection( std::make_shared(listener_addr), Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + EXPECT_NE(nullptr, client_conn); + EXPECT_TRUE(client_conn->connecting()); + client_conn->connect(); + // Connect returns error immediately because no internal listener manager is registered. + EXPECT_FALSE(client_conn->connecting()); client_conn->close(Network::ConnectionCloseType::NoFlush); } -TEST_F(ClientConnectionFactoryTest, BasicRecv) { - Buffer::OwnedImpl buf_to_write("0123456789"); - io_handle_peer_->write(buf_to_write); - { - auto result = io_handle_->recv(buf_.data(), buf_.size(), 0); - ASSERT_EQ(10, result.return_value_); - ASSERT_EQ("0123456789", absl::string_view(buf_.data(), result.return_value_)); - } - { - auto result = io_handle_->recv(buf_.data(), buf_.size(), 0); - // `EAGAIN`. - EXPECT_FALSE(result.ok()); - EXPECT_EQ(Api::IoError::IoErrorCode::Again, result.err_->getErrorCode()); - } - { - io_handle_->setWriteEnd(); - auto result = io_handle_->recv(buf_.data(), buf_.size(), 0); - EXPECT_TRUE(result.ok()); - } + +TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalListenerNotExist) { + MockInternalListenerManger internal_listener_manager; + dispatcher_->registerInternalListenerManager(internal_listener_manager); + + EXPECT_CALL(internal_listener_manager, findByAddress(_)) + .WillOnce(testing::Return(Network::InternalListenerCallbacksOptRef())); + + auto client_conn = dispatcher_->createClientConnection( + std::make_shared(listener_addr), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + + EXPECT_NE(nullptr, client_conn); + EXPECT_TRUE(client_conn->connecting()); + client_conn->connect(); + // Connect returns error immediately because no internal listener is ready. + EXPECT_FALSE(client_conn->connecting()); + client_conn->close(Network::ConnectionCloseType::NoFlush); } +// Verify that the client connection to envoy internal address can be established. This test case +// does not instantiate a server connection. The server connection is tested in internal listener. +TEST_F(ClientConnectionFactoryTest, ConnectSucceeds) { + MockInternalListenerManger internal_listener_manager; + dispatcher_->registerInternalListenerManager(internal_listener_manager); + MockInternalListenerCallbacks internal_listener; + Network::InternalListenerCallbacksOptRef internal_listener_opt = absl::make_optional( + std::reference_wrapper(internal_listener)); + + EXPECT_CALL(internal_listener_manager, findByAddress(_)) + .WillOnce(testing::Return(internal_listener_opt)); + Network::ConnectionSocketPtr server_socket; + EXPECT_CALL(internal_listener, onAccept(_)).WillOnce([&](auto&& socket) { + server_socket = std::move(socket); + }); + + auto client_conn = dispatcher_->createClientConnection( + std::make_shared(listener_addr), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + + EXPECT_NE(nullptr, server_socket); + + EXPECT_NE(nullptr, client_conn); + EXPECT_TRUE(client_conn->connecting()); + client_conn->connect(); + + // Connect is successful but the connecting state takes another poll cycle to clear. + EXPECT_TRUE(client_conn->connecting()); + + Buffer::OwnedImpl buf_to_write("0123456789"); + + client_conn->write(buf_to_write, false); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + // The write callback detects that connecting is completed. + EXPECT_FALSE(client_conn->connecting()); + + auto result = server_socket->ioHandle().recv(buf_.data(), buf_.size(), 0); + ASSERT_EQ(10, result.return_value_); + ASSERT_EQ("0123456789", absl::string_view(buf_.data(), result.return_value_)); + + client_conn->close(Network::ConnectionCloseType::NoFlush); + server_socket->close(); +} } // namespace } // namespace UserSpace } // namespace IoSocket } // namespace Extensions -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index a64ba2dab4b15..eb4b47df2fbd3 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -783,6 +783,9 @@ class FakeUpstream : Logger::Loggable, uint64_t listenerTag() const override { return 0; } const std::string& name() const override { return name_; } Network::UdpListenerConfigOptRef udpListenerConfig() override { return udp_listener_config_; } + Network::InternalListenerConfigOptRef internalListenerConfig() override { + return Network::InternalListenerConfigOptRef(); + } Network::ConnectionBalancer& connectionBalancer() override { return connection_balancer_; } envoy::config::core::v3::TrafficDirection direction() const override { return envoy::config::core::v3::UNSPECIFIED; diff --git a/test/mocks/event/wrapped_dispatcher.h b/test/mocks/event/wrapped_dispatcher.h index de0949358f5d7..d4ae6052bf551 100644 --- a/test/mocks/event/wrapped_dispatcher.h +++ b/test/mocks/event/wrapped_dispatcher.h @@ -50,6 +50,15 @@ class WrappedDispatcher : public Dispatcher { std::move(transport_socket), options); } + void registerInternalListenerManager( + Network::InternalListenerManager& internal_listener_manager) override { + impl_.registerInternalListenerManager(internal_listener_manager); + } + + Network::InternalListenerManagerOptRef getInternalListenerManager() override { + return impl_.getInternalListenerManager(); + } + Network::DnsResolverSharedPtr createDnsResolver( const std::vector& resolvers, const envoy::config::core::v3::DnsResolverOptions& dns_resolver_options) override { diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 22ea2a6d1f663..beabf64867e83 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -410,6 +410,7 @@ class MockListenerConfig : public ListenerConfig { MOCK_METHOD(uint64_t, listenerTag, (), (const)); MOCK_METHOD(const std::string&, name, (), (const)); MOCK_METHOD(Network::UdpListenerConfigOptRef, udpListenerConfig, ()); + MOCK_METHOD(Network::InternalListenerConfigOptRef, internalListenerConfig, ()); MOCK_METHOD(ConnectionBalancer&, connectionBalancer, ()); MOCK_METHOD(ResourceLimit&, openConnections, ()); MOCK_METHOD(uint32_t, tcpBacklogSize, (), (const)); diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 442c38adc9999..acebfecd2adaf 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -126,6 +126,9 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable Date: Wed, 3 Nov 2021 20:45:05 +0000 Subject: [PATCH 07/59] add default client connection factory Signed-off-by: Yuchen Dai --- envoy/network/BUILD | 1 + envoy/network/client_connection_manager.h | 3 ++- source/common/network/BUILD | 25 +++++++++++++++++++ .../default_client_connection_factory.cc | 24 ++++++++++++++++++ .../default_client_connection_factory.h | 23 +++++++++++++++++ source/server/connection_handler_impl.cc | 8 ++++++ source/server/connection_handler_impl.h | 5 ++++ source/server/worker_impl.cc | 1 + 8 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 source/common/network/default_client_connection_factory.cc create mode 100644 source/common/network/default_client_connection_factory.h diff --git a/envoy/network/BUILD b/envoy/network/BUILD index 0e2359933c4ee..ffb81409c52a0 100644 --- a/envoy/network/BUILD +++ b/envoy/network/BUILD @@ -39,6 +39,7 @@ envoy_cc_library( ":connection_interface", ":listen_socket_interface", ":transport_socket_interface", + "//envoy/config:typed_config_interface", ], ) diff --git a/envoy/network/client_connection_manager.h b/envoy/network/client_connection_manager.h index 8375dc8ae2583..f5a4fd260f633 100644 --- a/envoy/network/client_connection_manager.h +++ b/envoy/network/client_connection_manager.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/config/typed_config.h" #include "envoy/network/address.h" #include "envoy/network/connection.h" #include "envoy/network/listen_socket.h" @@ -8,7 +9,7 @@ namespace Envoy { namespace Network { -class ClientConnectionFactory { +class ClientConnectionFactory : public Config::UntypedFactory{ public: virtual ~ClientConnectionFactory() = default; std::string category() { return "network.connection"; } diff --git a/source/common/network/BUILD b/source/common/network/BUILD index b75b299dc7c8f..70b3e6a00fed7 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -79,6 +79,7 @@ envoy_cc_library( deps = [ ":address_lib", ":connection_base_lib", + ":default_client_connection_factory", ":raw_buffer_socket_lib", ":utility_lib", "//envoy/event:timer_interface", @@ -108,6 +109,30 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "default_client_connection_factory", + srcs = [ + "default_client_connection_factory.cc", + ], + hdrs = [ + "connection_impl.h", + "default_client_connection_factory.h", + ], + deps = [ + ":address_lib", + "//envoy/network:client_connection_manager", + "//envoy/network:connection_interface", + "//envoy/network:transport_socket_interface", + "//envoy/registry", + + # required by connection_impl.h + # will organize below when default client factory is confirmed to use dedicated target. + "//source/common/buffer:watermark_buffer_lib", + ":connection_base_lib", + "//source/common/stream_info:stream_info_lib", + ], +) + envoy_cc_library( name = "filter_lib", hdrs = ["filter_impl.h"], diff --git a/source/common/network/default_client_connection_factory.cc b/source/common/network/default_client_connection_factory.cc new file mode 100644 index 0000000000000..e1a65a0e81579 --- /dev/null +++ b/source/common/network/default_client_connection_factory.cc @@ -0,0 +1,24 @@ +#include "source/common/network/default_client_connection_factory.h" + +#include "envoy/registry/registry.h" + +#include "source/common/network/address_impl.h" +#include "source/common/network/connection_impl.h" + +namespace Envoy { + +namespace Network { + +Network::ClientConnectionPtr DefaultClientConnectionFactory::createClientConnection( + Event::Dispatcher& dispatcher, Network::Address::InstanceConstSharedPtr address, + Network::Address::InstanceConstSharedPtr source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) { + ASSERT(address->ip() || address->pipe()); + return std::make_unique(dispatcher, address, source_address, + std::move(transport_socket), options); +} +REGISTER_FACTORY(DefaultClientConnectionFactory, Network::ClientConnectionFactory); + +} // namespace Network +} // namespace Envoy diff --git a/source/common/network/default_client_connection_factory.h b/source/common/network/default_client_connection_factory.h new file mode 100644 index 0000000000000..2de29bd259ae6 --- /dev/null +++ b/source/common/network/default_client_connection_factory.h @@ -0,0 +1,23 @@ +#pragma once + +#include "envoy/common/pure.h" +#include "envoy/network/client_connection_manager.h" +#include "envoy/network/connection.h" + +namespace Envoy { + +namespace Network { + +class DefaultClientConnectionFactory : public Network::ClientConnectionFactory { +public: + std::string name() override { return "default"; } + Network::ClientConnectionPtr + createClientConnection(Event::Dispatcher& dispatcher, + Network::Address::InstanceConstSharedPtr address, + Network::Address::InstanceConstSharedPtr source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) override; +}; + +} // namespace Network +} // namespace Envoy diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index 4bcdcd9cc16aa..da74f75e07f1c 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -241,5 +241,13 @@ ConnectionHandlerImpl::getBalancedHandlerByAddress(const Network::Address::Insta : absl::nullopt; } +// Network::InternalManager +Network::InternalListenerCallbacksOptRef ConnectionHandlerImpl::findByAddress( + const Network::Address::InstanceConstSharedPtr& listen_address) { + // TODO(lambdai): lookup the internal listene when #18104 is merged. + UNREFERENCED_PARAMETER(listen_address); + return Network::InternalListenerCallbacksOptRef(); +} + } // namespace Server } // namespace Envoy diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index f10424b7c1a2d..81726dd7cfdb3 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -28,6 +28,7 @@ class ActiveTcpListener; */ class ConnectionHandlerImpl : public Network::TcpConnectionHandler, public Network::UdpConnectionHandler, + public Network::InternalListenerManager, NonCopyable, Logger::Loggable { public: @@ -63,6 +64,10 @@ class ConnectionHandlerImpl : public Network::TcpConnectionHandler, // Network::UdpConnectionHandler Network::UdpListenerCallbacksOptRef getUdpListenerCallbacks(uint64_t listener_tag) override; + // Network::InternalManager + Network::InternalListenerCallbacksOptRef + findByAddress(const Network::Address::InstanceConstSharedPtr& listen_address) override; + private: struct ActiveListenerDetails { // Strong pointer to the listener, whether TCP, UDP, QUIC, etc. diff --git a/source/server/worker_impl.cc b/source/server/worker_impl.cc index 3e0fdbc93e2bc..bbadce5ce79fd 100644 --- a/source/server/worker_impl.cc +++ b/source/server/worker_impl.cc @@ -19,6 +19,7 @@ WorkerPtr ProdWorkerFactory::createWorker(uint32_t index, OverloadManager& overl Event::DispatcherPtr dispatcher( api_.allocateDispatcher(worker_name, overload_manager.scaledTimerFactory())); auto conn_handler = std::make_unique(*dispatcher, index); + dispatcher->registerInternalListenerManager(*conn_handler); return std::make_unique(tls_, hooks_, std::move(dispatcher), std::move(conn_handler), overload_manager, api_, stat_names_); } From b94abcb2205a9ae8b631787e08124cecbf19d35d Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 5 Nov 2021 21:56:48 +0000 Subject: [PATCH 08/59] adding getFactoryByAddressType Signed-off-by: Yuchen Dai --- source/common/event/dispatcher_impl.cc | 3 +-- source/common/network/BUILD | 1 + source/common/network/address_impl.cc | 7 +++++++ source/common/network/address_impl.h | 5 ++++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 251f31c7c76a4..240f0e87dab58 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -163,8 +163,7 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr } }; // TODO: register and find by address type instead of name. - auto factory = Config::Utility::getFactoryByName( - address_type_name(address)); + auto factory = Config::Utility::getFactoryByAddressType(address->type()); if (factory == nullptr) { // get rid of this once the ip and pipe factory is offered. return std::make_unique(*this, address, source_address, diff --git a/source/common/network/BUILD b/source/common/network/BUILD index 70b3e6a00fed7..944de4e158a25 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -15,6 +15,7 @@ envoy_cc_library( deps = [ ":socket_interface_lib", "//envoy/network:address_interface", + "//envoy/network:client_connection_manager", "//source/common/api:os_sys_calls_lib", "//source/common/common:assert_lib", "//source/common/common:safe_memcpy_lib", diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 5954724186bce..34906c7ba9e20 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -118,6 +118,13 @@ addressFromSockAddrOrDie(const sockaddr_storage& ss, socklen_t ss_len, os_fd_t f return *address; } +Network::ClientConnectionFactory* getFactoryAddressType(Address::Type address_type) { + switch (address_type) { + + } +} + + Ipv4Instance::Ipv4Instance(const sockaddr_in* address, const SocketInterface* sock_interface) : InstanceBase(Type::Ip, sockInterfaceOrDefault(sock_interface)) { throwOnError(validateProtocolSupported()); diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index 2ade85fc89984..c33870bd03148 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -9,6 +9,7 @@ #include "envoy/common/platform.h" #include "envoy/network/address.h" #include "envoy/network/socket.h" +#include "envoy/network/client_connection_manager.h" #include "source/common/common/assert.h" #include "source/common/common/statusor.h" @@ -31,6 +32,9 @@ StatusOr addressFromSockAddr(const sockaddr_storage& ss, InstanceConstSharedPtr addressFromSockAddrOrThrow(const sockaddr_storage& ss, socklen_t len, bool v6only = true); +Network::ClientConnectionFactory* getFactoryAddressType(Address::Type address_type); + + /** * Convert an address in the form of the socket address struct defined by Posix, Linux, etc. into * a Network::Address::Instance and return a pointer to it. Die on failure. @@ -354,7 +358,6 @@ class EnvoyInternalInstance : public InstanceBase { }; EnvoyInternalAddressImpl internal_address_; }; - } // namespace Address } // namespace Network } // namespace Envoy From 85e3a73d8346e085b1997a05a94bd6386f232955 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 5 Nov 2021 21:57:00 +0000 Subject: [PATCH 09/59] Revert "adding getFactoryByAddressType" This reverts commit b94abcb2205a9ae8b631787e08124cecbf19d35d. Signed-off-by: Yuchen Dai --- source/common/event/dispatcher_impl.cc | 3 ++- source/common/network/BUILD | 1 - source/common/network/address_impl.cc | 7 ------- source/common/network/address_impl.h | 5 +---- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 240f0e87dab58..251f31c7c76a4 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -163,7 +163,8 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr } }; // TODO: register and find by address type instead of name. - auto factory = Config::Utility::getFactoryByAddressType(address->type()); + auto factory = Config::Utility::getFactoryByName( + address_type_name(address)); if (factory == nullptr) { // get rid of this once the ip and pipe factory is offered. return std::make_unique(*this, address, source_address, diff --git a/source/common/network/BUILD b/source/common/network/BUILD index 944de4e158a25..70b3e6a00fed7 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -15,7 +15,6 @@ envoy_cc_library( deps = [ ":socket_interface_lib", "//envoy/network:address_interface", - "//envoy/network:client_connection_manager", "//source/common/api:os_sys_calls_lib", "//source/common/common:assert_lib", "//source/common/common:safe_memcpy_lib", diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 34906c7ba9e20..5954724186bce 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -118,13 +118,6 @@ addressFromSockAddrOrDie(const sockaddr_storage& ss, socklen_t ss_len, os_fd_t f return *address; } -Network::ClientConnectionFactory* getFactoryAddressType(Address::Type address_type) { - switch (address_type) { - - } -} - - Ipv4Instance::Ipv4Instance(const sockaddr_in* address, const SocketInterface* sock_interface) : InstanceBase(Type::Ip, sockInterfaceOrDefault(sock_interface)) { throwOnError(validateProtocolSupported()); diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index c33870bd03148..2ade85fc89984 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -9,7 +9,6 @@ #include "envoy/common/platform.h" #include "envoy/network/address.h" #include "envoy/network/socket.h" -#include "envoy/network/client_connection_manager.h" #include "source/common/common/assert.h" #include "source/common/common/statusor.h" @@ -32,9 +31,6 @@ StatusOr addressFromSockAddr(const sockaddr_storage& ss, InstanceConstSharedPtr addressFromSockAddrOrThrow(const sockaddr_storage& ss, socklen_t len, bool v6only = true); -Network::ClientConnectionFactory* getFactoryAddressType(Address::Type address_type); - - /** * Convert an address in the form of the socket address struct defined by Posix, Linux, etc. into * a Network::Address::Instance and return a pointer to it. Die on failure. @@ -358,6 +354,7 @@ class EnvoyInternalInstance : public InstanceBase { }; EnvoyInternalAddressImpl internal_address_; }; + } // namespace Address } // namespace Network } // namespace Envoy From 2adc021cebb8fc80c0020cd073970b4a1c22fa56 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 8 Nov 2021 23:43:51 +0000 Subject: [PATCH 10/59] client connection factory follows UntypedFactory Signed-off-by: Yuchen Dai --- envoy/event/dispatcher.h | 2 +- envoy/network/client_connection_manager.h | 8 ++-- source/common/event/BUILD | 5 +- source/common/event/dispatcher_impl.cc | 21 ++------ source/common/network/address_impl.cc | 11 +++++ source/common/network/address_impl.h | 6 +++ .../default_client_connection_factory.h | 3 +- .../user_space/client_connection_factory.cc | 48 +++++++------------ .../user_space/client_connection_factory.h | 8 +++- 9 files changed, 54 insertions(+), 58 deletions(-) diff --git a/envoy/event/dispatcher.h b/envoy/event/dispatcher.h index 0d6f12ebfb472..c8328f04911e0 100644 --- a/envoy/event/dispatcher.h +++ b/envoy/event/dispatcher.h @@ -226,7 +226,7 @@ class Dispatcher : public DispatcherBase, public ScopeTracker { /** * @brief Get the Internal Listener Manager object. - * + * * @return the registered internal istener manager or nullopt. */ virtual Network::InternalListenerManagerOptRef getInternalListenerManager() PURE; diff --git a/envoy/network/client_connection_manager.h b/envoy/network/client_connection_manager.h index f5a4fd260f633..12d2d5b9956f6 100644 --- a/envoy/network/client_connection_manager.h +++ b/envoy/network/client_connection_manager.h @@ -9,11 +9,13 @@ namespace Envoy { namespace Network { -class ClientConnectionFactory : public Config::UntypedFactory{ +class ClientConnectionFactory : public Config::UntypedFactory { public: + ClientConnectionFactory() = default; virtual ~ClientConnectionFactory() = default; - std::string category() { return "network.connection"; } - virtual std::string name() PURE; + + // Config::UntypedFactory + std::string category() const override { return "network.connection"; } virtual Network::ClientConnectionPtr createClientConnection(Event::Dispatcher& dispatcher, diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 99f763231177c..5038704d5192d 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -31,8 +31,6 @@ envoy_cc_library( "//conditions:default": "posix", }), deps = [ - "//envoy/network:client_connection_manager", - "//source/common/config:utility_lib", ":dispatcher_includes", ":libevent_scheduler_lib", ":real_time_system_lib", @@ -41,11 +39,14 @@ envoy_cc_library( "//envoy/common:scope_tracker_interface", "//envoy/common:time_interface", "//envoy/event:signal_interface", + "//envoy/network:client_connection_manager", "//envoy/network:listen_socket_interface", "//envoy/network:listener_interface", "//source/common/common:assert_lib", "//source/common/common:thread_lib", + "//source/common/config:utility_lib", "//source/common/filesystem:watcher_lib", + "//source/common/network:address_lib", "//source/common/network:connection_lib", "//source/common/network:listener_lib", "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 251f31c7c76a4..91448012b4eaa 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -24,6 +24,7 @@ #include "source/common/event/signal_impl.h" #include "source/common/event/timer_impl.h" #include "source/common/filesystem/watcher_impl.h" +#include "source/common/network/address_impl.h" #include "source/common/network/connection_impl.h" #include "source/common/network/tcp_listener_impl.h" #include "source/common/network/udp_listener_impl.h" @@ -151,25 +152,9 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr const Network::ConnectionSocket::OptionsSharedPtr& options) { ASSERT(isThreadSafe()); - auto address_type_name = [](const Network::Address::InstanceConstSharedPtr& addr) { - ASSERT(addr != nullptr); - switch (addr->type()) { - // TODO: create IP and - case Network::Address::Type::Ip: - case Network::Address::Type::Pipe: - return "does-not-exist"; - case Network::Address::Type::EnvoyInternal: - return "EnvoyInternal"; - } - }; - // TODO: register and find by address type instead of name. auto factory = Config::Utility::getFactoryByName( - address_type_name(address)); - if (factory == nullptr) { - // get rid of this once the ip and pipe factory is offered. - return std::make_unique(*this, address, source_address, - std::move(transport_socket), options); - } + std::string(Network::Address::addressType(address))); + // The caller expects a non-null connection as of today. A unsupported address will crash any way. return factory->createClientConnection(*this, address, source_address, std::move(transport_socket), options); } diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 5954724186bce..c78ee34964d44 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -118,6 +118,17 @@ addressFromSockAddrOrDie(const sockaddr_storage& ss, socklen_t ss_len, os_fd_t f return *address; } +absl::string_view addressType(const Network::Address::InstanceConstSharedPtr& addr) { + ASSERT(addr != nullptr); + switch (addr->type()) { + case Network::Address::Type::Ip: + case Network::Address::Type::Pipe: + return "default"; + case Network::Address::Type::EnvoyInternal: + return "EnvoyInternal"; + } +}; + Ipv4Instance::Ipv4Instance(const sockaddr_in* address, const SocketInterface* sock_interface) : InstanceBase(Type::Ip, sockInterfaceOrDefault(sock_interface)) { throwOnError(validateProtocolSupported()); diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index 2ade85fc89984..76bf59ccddadc 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -44,6 +44,12 @@ InstanceConstSharedPtr addressFromSockAddrOrThrow(const sockaddr_storage& ss, so InstanceConstSharedPtr addressFromSockAddrOrDie(const sockaddr_storage& ss, socklen_t ss_len, os_fd_t fd, bool v6only = true); +/** + * Return the address type in string_view. The returned type name is used by calling + * ClientConnectionFactory. + */ +absl::string_view addressType(const Network::Address::InstanceConstSharedPtr& addr); + /** * Base class for all address types. */ diff --git a/source/common/network/default_client_connection_factory.h b/source/common/network/default_client_connection_factory.h index 2de29bd259ae6..de78491df9d20 100644 --- a/source/common/network/default_client_connection_factory.h +++ b/source/common/network/default_client_connection_factory.h @@ -10,7 +10,8 @@ namespace Network { class DefaultClientConnectionFactory : public Network::ClientConnectionFactory { public: - std::string name() override { return "default"; } + ~DefaultClientConnectionFactory() = default; + std::string name() const override { return "default"; } Network::ClientConnectionPtr createClientConnection(Event::Dispatcher& dispatcher, Network::Address::InstanceConstSharedPtr address, diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index 91f211376dc97..334d5daa705e2 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -19,10 +19,7 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) { - Network::IoHandlePtr io_handle_client; - Network::IoHandlePtr io_handle_server; - - std::tie(io_handle_client, io_handle_server) = + auto [io_handle_client, io_handle_server] = Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); auto client_conn = std::make_unique( @@ -30,38 +27,27 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec std::make_unique(std::move(io_handle_client), source_address, address), source_address, std::move(transport_socket), options); - // TODO(lambdai): refactor nested if. + + // It's either in the main thread or the worker is not yet started. auto internal_listener_manager = dispatcher.getInternalListenerManager(); - if (internal_listener_manager.has_value()) { - // It's either in main thread or the worker is not yet started. - auto internal_listener = internal_listener_manager.value().get().findByAddress(address); - if (internal_listener.has_value()) { - auto original_address = address; - // if (options != nullptr) { - // for (const auto& opt : *options) { - // auto* internal_opt = dynamic_cast(opt.get()); - // if (internal_opt != nullptr) { - // original_address = internal_opt->original_remote_address_; - // } - // } - // } - auto accepted_socket = std::make_unique( - std::move(io_handle_server), original_address, source_address); - // TODO: also check if disabled - internal_listener.value().get().onAccept(std::move(accepted_socket)); - FANCY_LOG(debug, "lambdai: find internal listener {} ", address->asStringView()); - } else { - FANCY_LOG(debug, "lambdai: cannot find internal listener {} ", address->asStringView()); - // injected error into client_conn; - io_handle_server->close(); - } - } else { - FANCY_LOG(debug, "lambdai: cannot find internal listener {} ", address->asStringView()); - // injected error into client_conn; + if (!internal_listener_manager.has_value()) { + io_handle_server->close(); + return client_conn; + } + + // The request internal listener may not exist. + auto internal_listener = internal_listener_manager.value().get().findByAddress(address); + if (!internal_listener.has_value()) { io_handle_server->close(); + return client_conn; } + + auto accepted_socket = std::make_unique(std::move(io_handle_server), + address, source_address); + internal_listener->get().onAccept(std::move(accepted_socket)); return client_conn; } + REGISTER_FACTORY(InternalClientConnectionFactory, Network::ClientConnectionFactory); } // namespace UserSpace diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/io_socket/user_space/client_connection_factory.h index 5ea485178a9eb..050bbb87962b6 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.h +++ b/source/extensions/io_socket/user_space/client_connection_factory.h @@ -7,15 +7,19 @@ #include "envoy/network/client_connection_manager.h" #include "envoy/network/connection.h" +#include "source/common/common/logger.h" + namespace Envoy { namespace Extensions { namespace IoSocket { namespace UserSpace { -class InternalClientConnectionFactory : public Network::ClientConnectionFactory { +class InternalClientConnectionFactory : public Network::ClientConnectionFactory, + Logger::Loggable { public: - std::string name() override { return "EnvoyInternal"; } + ~InternalClientConnectionFactory() = default; + std::string name() const override { return "EnvoyInternal"; } Network::ClientConnectionPtr createClientConnection(Event::Dispatcher& dispatcher, Network::Address::InstanceConstSharedPtr address, From bcb375b0d4f0c3b62b0b23406ba0f1de4b1f8178 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 29 Nov 2021 19:13:59 +0000 Subject: [PATCH 11/59] fix after merge Signed-off-by: Yuchen Dai --- envoy/network/listener.h | 29 ------------------- .../user_space/client_connection_factory.cc | 4 +-- source/server/connection_handler_impl.cc | 8 ----- test/common/event/dispatcher_impl_test.cc | 2 +- test/common/network/connection_impl_test.cc | 7 +++-- .../client_connection_factory_test.cc | 11 ++++--- .../socket_interface_integration_test.cc | 5 ++-- 7 files changed, 16 insertions(+), 50 deletions(-) diff --git a/envoy/network/listener.h b/envoy/network/listener.h index 62f7dc813e4ac..fac995eb3ed6f 100644 --- a/envoy/network/listener.h +++ b/envoy/network/listener.h @@ -448,35 +448,6 @@ class UdpListener : public virtual Listener { }; using UdpListenerPtr = std::unique_ptr; -class InternalListenerCallbacks { -public: - virtual ~InternalListenerCallbacks() = default; - - /** - * Called when a new connection is accepted. - * @param socket supplies the socket that is moved into the callee. - */ - virtual void onAccept(ConnectionSocketPtr&& socket) PURE; - - virtual Event::Dispatcher& dispatcher() PURE; -}; -using InternalListenerCallbacksOptRef = - absl::optional>; - -class InternalListener {}; - -using InternalListenerPtr = std::unique_ptr; -using InternalListenerOptRef = absl::optional>; - -class InternalListenerManager { -public: - virtual ~InternalListenerManager() = default; - virtual InternalListenerCallbacksOptRef - findByAddress(const Address::InstanceConstSharedPtr& listen_address) PURE; -}; - -using InternalListenerManagerOptRef = - absl::optional>; /** * Internal listener callbacks. diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index 334d5daa705e2..7fb7fdee59bc7 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -34,7 +34,7 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec io_handle_server->close(); return client_conn; } - + // The request internal listener may not exist. auto internal_listener = internal_listener_manager.value().get().findByAddress(address); if (!internal_listener.has_value()) { @@ -44,7 +44,7 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec auto accepted_socket = std::make_unique(std::move(io_handle_server), address, source_address); - internal_listener->get().onAccept(std::move(accepted_socket)); + internal_listener->onAccept(std::move(accepted_socket)); return client_conn; } diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index 920d5475fa48a..24789e5feb5aa 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -281,13 +281,5 @@ ConnectionHandlerImpl::getBalancedHandlerByAddress(const Network::Address::Insta : absl::nullopt; } -// Network::InternalManager -Network::InternalListenerCallbacksOptRef ConnectionHandlerImpl::findByAddress( - const Network::Address::InstanceConstSharedPtr& listen_address) { - // TODO(lambdai): lookup the internal listene when #18104 is merged. - UNREFERENCED_PARAMETER(listen_address); - return Network::InternalListenerCallbacksOptRef(); -} - } // namespace Server } // namespace Envoy diff --git a/test/common/event/dispatcher_impl_test.cc b/test/common/event/dispatcher_impl_test.cc index 83a32175af04e..ac979a379f6ef 100644 --- a/test/common/event/dispatcher_impl_test.cc +++ b/test/common/event/dispatcher_impl_test.cc @@ -1578,7 +1578,7 @@ TEST_F(DispatcherConnectionTest, CreateEnvoyInternalConnectionWhenFactoryNotExis std::make_shared("listener_internal_address"), Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr), - "not implemented"); + ""); } } // namespace diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 88b8f79e9e6b7..c0647d09bd0a3 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -2989,6 +2989,9 @@ class InternalClientConnectionImplTest : public testing::Test { StrictMock client_callbacks_; }; +// The internal address is passed to Envoy by EDS. If this Envoy instance is configured as internal +// address disabled, the EDS subscription should reject the config before dispatcher attempt to +// establish connection to such address. TEST_F(InternalClientConnectionImplTest, CannotCreateConnectionToInternalAddressWithInternalAddressEnabled) { auto scoped_runtime_guard = std::make_unique(); @@ -2999,14 +3002,14 @@ TEST_F(InternalClientConnectionImplTest, "envoy.extensions.network.socket_interface.default_socket_interface"); Network::Address::InstanceConstSharedPtr address = std::make_shared("listener_0", sock_interface); - // Not implemented yet. + ASSERT_DEATH( { ClientConnectionPtr connection = dispatcher_->createClientConnection(address, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); }, - "panic: not implemented"); + ""); } } // namespace diff --git a/test/extensions/io_socket/user_space/client_connection_factory_test.cc b/test/extensions/io_socket/user_space/client_connection_factory_test.cc index a7b55c1476053..d9f321ab6360f 100644 --- a/test/extensions/io_socket/user_space/client_connection_factory_test.cc +++ b/test/extensions/io_socket/user_space/client_connection_factory_test.cc @@ -45,7 +45,7 @@ class ClientConnectionFactoryTest : public testing::Test { Network::Address::EnvoyInternalInstance listener_addr{"listener_internal_address"}; }; -class MockInternalListenerCallbacks : public Network::InternalListenerCallbacks { +class MockInternalListener : public Network::InternalListener { public: MOCK_METHOD(void, onAccept, (Network::ConnectionSocketPtr &&)); MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); @@ -53,7 +53,7 @@ class MockInternalListenerCallbacks : public Network::InternalListenerCallbacks class MockInternalListenerManger : public Network::InternalListenerManager { public: - MOCK_METHOD(Network::InternalListenerCallbacksOptRef, findByAddress, + MOCK_METHOD(Network::InternalListenerOptRef, findByAddress, (const Network::Address::InstanceConstSharedPtr&)); }; @@ -74,7 +74,7 @@ TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalListenerNotExist) { dispatcher_->registerInternalListenerManager(internal_listener_manager); EXPECT_CALL(internal_listener_manager, findByAddress(_)) - .WillOnce(testing::Return(Network::InternalListenerCallbacksOptRef())); + .WillOnce(testing::Return(Network::InternalListenerOptRef())); auto client_conn = dispatcher_->createClientConnection( std::make_shared(listener_addr), @@ -93,9 +93,8 @@ TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalListenerNotExist) { TEST_F(ClientConnectionFactoryTest, ConnectSucceeds) { MockInternalListenerManger internal_listener_manager; dispatcher_->registerInternalListenerManager(internal_listener_manager); - MockInternalListenerCallbacks internal_listener; - Network::InternalListenerCallbacksOptRef internal_listener_opt = absl::make_optional( - std::reference_wrapper(internal_listener)); + MockInternalListener internal_listener; + Network::InternalListenerOptRef internal_listener_opt{internal_listener}; EXPECT_CALL(internal_listener_manager, findByAddress(_)) .WillOnce(testing::Return(internal_listener_opt)); diff --git a/test/integration/socket_interface_integration_test.cc b/test/integration/socket_interface_integration_test.cc index 9cae29698a006..c0a4c987056e8 100644 --- a/test/integration/socket_interface_integration_test.cc +++ b/test/integration/socket_interface_integration_test.cc @@ -89,7 +89,8 @@ TEST_P(SocketInterfaceIntegrationTest, AddressWithSocketInterface) { client_->close(Network::ConnectionCloseType::FlushWrite); } -// Test that connecting to internal address will crash. +// Test that connecting to internal address will crash if the user space socket extension is not +// linked. TEST_P(SocketInterfaceIntegrationTest, InternalAddressWithSocketInterface) { BaseIntegrationTest::initialize(); @@ -103,7 +104,7 @@ TEST_P(SocketInterfaceIntegrationTest, InternalAddressWithSocketInterface) { ASSERT_DEATH(client_ = dispatcher_->createClientConnection( address, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr), - "panic: not implemented"); + "" /* Nullptr dereference */); } // Test that recv from internal address will crash. From 83c193d9b3945709463b84f354615089fc61e7e6 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 29 Nov 2021 21:09:46 +0000 Subject: [PATCH 12/59] fix gcc switch enum Signed-off-by: Yuchen Dai --- source/common/network/address_impl.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index c78ee34964d44..5223e9cd5c564 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -127,6 +127,7 @@ absl::string_view addressType(const Network::Address::InstanceConstSharedPtr& ad case Network::Address::Type::EnvoyInternal: return "EnvoyInternal"; } + NOT_REACHED_GCOVR_EXCL_LINE; }; Ipv4Instance::Ipv4Instance(const sockaddr_in* address, const SocketInterface* sock_interface) From a932479af18965a8639d7488d393a77ef6506dc9 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 29 Nov 2021 23:20:05 +0000 Subject: [PATCH 13/59] clang-tidy and better name Signed-off-by: Yuchen Dai --- envoy/network/BUILD | 4 ++-- source/common/event/BUILD | 2 +- source/common/event/dispatcher_impl.cc | 2 +- source/common/network/BUILD | 2 +- source/common/network/default_client_connection_factory.h | 4 ++-- source/extensions/io_socket/user_space/BUILD | 2 +- .../io_socket/user_space/client_connection_factory.h | 4 ++-- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/envoy/network/BUILD b/envoy/network/BUILD index 422b9fa9fb58a..178009c63fa28 100644 --- a/envoy/network/BUILD +++ b/envoy/network/BUILD @@ -29,8 +29,8 @@ envoy_cc_library( ) envoy_cc_library( - name = "client_connection_manager", - hdrs = ["client_connection_manager.h"], + name = "client_connection_factory", + hdrs = ["client_connection_factory.h"], deps = [ ":address_interface", ":connection_interface", diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 5038704d5192d..8a1dafbe880e8 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -39,7 +39,7 @@ envoy_cc_library( "//envoy/common:scope_tracker_interface", "//envoy/common:time_interface", "//envoy/event:signal_interface", - "//envoy/network:client_connection_manager", + "//envoy/network:client_connection_factory", "//envoy/network:listen_socket_interface", "//envoy/network:listener_interface", "//source/common/common:assert_lib", diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 3f1f8d570fa73..2b45f13601586 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -9,7 +9,7 @@ #include "envoy/api/api.h" #include "envoy/common/scope_tracker.h" #include "envoy/config/overload/v3/overload.pb.h" -#include "envoy/network/client_connection_manager.h" +#include "envoy/network/client_connection_factory.h" #include "envoy/network/listen_socket.h" #include "envoy/network/listener.h" diff --git a/source/common/network/BUILD b/source/common/network/BUILD index ffbe437f6fb90..dea4dbe2d3da1 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -120,7 +120,7 @@ envoy_cc_library( ], deps = [ ":address_lib", - "//envoy/network:client_connection_manager", + "//envoy/network:client_connection_factory", "//envoy/network:connection_interface", "//envoy/network:transport_socket_interface", "//envoy/registry", diff --git a/source/common/network/default_client_connection_factory.h b/source/common/network/default_client_connection_factory.h index de78491df9d20..a4f96e29e267b 100644 --- a/source/common/network/default_client_connection_factory.h +++ b/source/common/network/default_client_connection_factory.h @@ -1,7 +1,7 @@ #pragma once #include "envoy/common/pure.h" -#include "envoy/network/client_connection_manager.h" +#include "envoy/network/client_connection_factory.h" #include "envoy/network/connection.h" namespace Envoy { @@ -10,7 +10,7 @@ namespace Network { class DefaultClientConnectionFactory : public Network::ClientConnectionFactory { public: - ~DefaultClientConnectionFactory() = default; + ~DefaultClientConnectionFactory() override = default; std::string name() const override { return "default"; } Network::ClientConnectionPtr createClientConnection(Event::Dispatcher& dispatcher, diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 1d4a144362202..402e99b524646 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -69,7 +69,7 @@ envoy_cc_library( ], deps = [ ":io_handle_impl_lib", - "//envoy/network:client_connection_manager", + "//envoy/network:client_connection_factory", "//envoy/network:connection_interface", "//envoy/registry", "//source/common/network:connection_lib", diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/io_socket/user_space/client_connection_factory.h index 050bbb87962b6..d143955b18273 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.h +++ b/source/extensions/io_socket/user_space/client_connection_factory.h @@ -4,7 +4,7 @@ #include #include "envoy/common/pure.h" -#include "envoy/network/client_connection_manager.h" +#include "envoy/network/client_connection_factory.h" #include "envoy/network/connection.h" #include "source/common/common/logger.h" @@ -18,7 +18,7 @@ namespace UserSpace { class InternalClientConnectionFactory : public Network::ClientConnectionFactory, Logger::Loggable { public: - ~InternalClientConnectionFactory() = default; + ~InternalClientConnectionFactory() override = default; std::string name() const override { return "EnvoyInternal"; } Network::ClientConnectionPtr createClientConnection(Event::Dispatcher& dispatcher, From a42efa6dc5c6be836ad9705af785994457df5835 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 30 Nov 2021 00:38:14 +0000 Subject: [PATCH 14/59] renaming Signed-off-by: Yuchen Dai --- ...{client_connection_manager.h => client_connection_factory.h} | 2 +- envoy/network/client_connection_manager.cc | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) rename envoy/network/{client_connection_manager.h => client_connection_factory.h} (94%) delete mode 100644 envoy/network/client_connection_manager.cc diff --git a/envoy/network/client_connection_manager.h b/envoy/network/client_connection_factory.h similarity index 94% rename from envoy/network/client_connection_manager.h rename to envoy/network/client_connection_factory.h index 12d2d5b9956f6..d94c4a0c1dc61 100644 --- a/envoy/network/client_connection_manager.h +++ b/envoy/network/client_connection_factory.h @@ -12,7 +12,7 @@ namespace Network { class ClientConnectionFactory : public Config::UntypedFactory { public: ClientConnectionFactory() = default; - virtual ~ClientConnectionFactory() = default; + virtual ~ClientConnectionFactory() override = default; // Config::UntypedFactory std::string category() const override { return "network.connection"; } diff --git a/envoy/network/client_connection_manager.cc b/envoy/network/client_connection_manager.cc deleted file mode 100644 index fafc4c928ff93..0000000000000 --- a/envoy/network/client_connection_manager.cc +++ /dev/null @@ -1 +0,0 @@ -// NOLINT(namespace-envoy) From 4caa42b39fab4e4442d0f8ad1ad84ee81ca99617 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 1 Dec 2021 18:23:26 +0000 Subject: [PATCH 15/59] tidy Signed-off-by: Yuchen Dai --- envoy/network/client_connection_factory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envoy/network/client_connection_factory.h b/envoy/network/client_connection_factory.h index d94c4a0c1dc61..142f4f54b7df7 100644 --- a/envoy/network/client_connection_factory.h +++ b/envoy/network/client_connection_factory.h @@ -12,7 +12,7 @@ namespace Network { class ClientConnectionFactory : public Config::UntypedFactory { public: ClientConnectionFactory() = default; - virtual ~ClientConnectionFactory() override = default; + ~ClientConnectionFactory() override = default; // Config::UntypedFactory std::string category() const override { return "network.connection"; } From 17afcbafaa5fbf647dae6646c244a17b60c8dbe5 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 15 Dec 2021 00:04:23 +0000 Subject: [PATCH 16/59] add internal listener registry as bootstrap extension Signed-off-by: Yuchen Dai --- source/extensions/io_socket/user_space/BUILD | 13 +++++++ .../user_space/internal_listener_registry.cc | 18 +++++++++ .../user_space/internal_listener_registry.h | 39 +++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 source/extensions/io_socket/user_space/internal_listener_registry.cc create mode 100644 source/extensions/io_socket/user_space/internal_listener_registry.h diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 402e99b524646..895488cfe6708 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -76,3 +76,16 @@ envoy_cc_library( "//source/common/network:listen_socket_lib", ], ) + +envoy_cc_library( + name = "internal_listener_registry", + srcs = [ + "internal_listener_registry.cc", + ], + hdrs = [ + "internal_listener_registry.h", + ], + deps = [ + "//source/server:bootstrap_extension_config_interface", + ] +) diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.cc b/source/extensions/io_socket/user_space/internal_listener_registry.cc new file mode 100644 index 0000000000000..fbcc33db5b9e3 --- /dev/null +++ b/source/extensions/io_socket/user_space/internal_listener_registry.cc @@ -0,0 +1,18 @@ +#include "source/extensions/io_socket/user_space/internal_listener_registry.h" + +namespace Envoy { +namespace Extensions { +namespace Bootstrap { +namespace InternalListener { + +void InternalListenerExtension::onServerInitialized() {} + +Server::BootstrapExtensionPtr InternalListenerRegisteryFactory::createBootstrapExtension( + const Protobuf::Message&, Server::Configuration::ServerFactoryContext&) { + return nullptr; +} + +} // namespace InternalListener +} // namespace Bootstrap +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.h b/source/extensions/io_socket/user_space/internal_listener_registry.h new file mode 100644 index 0000000000000..9fa6f9c2346d3 --- /dev/null +++ b/source/extensions/io_socket/user_space/internal_listener_registry.h @@ -0,0 +1,39 @@ +#pragma once + +#include "envoy/config/typed_config.h" +#include "envoy/registry/registry.h" +#include "envoy/server/bootstrap_extension_config.h" + +#include "source/common/singleton/threadsafe_singleton.h" + +#include "absl/container/flat_hash_map.h" + +namespace Envoy { +namespace Extensions { +namespace Bootstrap { +namespace InternalListener { + +class InternalListenerExtension : public Server::Configuration::BootstrapExtension { +public: + ~InternalListenerExtension() override = default; + + // Server::Configuration::BootstrapExtension + void onServerInitialized() override; +}; + +class InternalListenerRegisteryFactory : public Server::Configuration::BootstrapExtensionFactory { +public: + // Server::Configuration::BootstrapExtensionFactory + Server::BootstrapExtensionPtr + createBootstrapExtension(const Protobuf::Message& config, + Server::Configuration::ServerFactoryContext& context) override; + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + std::string name() const override { return "envoy.bootstrap.internal_listener"; }; +}; + +} // namespace InternalListener +} // namespace Bootstrap +} // namespace Extensions +} // namespace Envoy From f67542ae9cf9f662e18f58f3aec485b2ce5e04a4 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 20 Dec 2021 23:00:27 +0000 Subject: [PATCH 17/59] introduce singleton helper to create bind tls internal listener registry Signed-off-by: Yuchen Dai --- envoy/network/listener.h | 19 ++++++++++ source/extensions/io_socket/user_space/BUILD | 24 +++++++++++-- .../user_space/client_connection_factory.cc | 3 ++ .../user_space/client_connection_factory.h | 10 ++++++ .../user_space/internal_listener_registry.cc | 30 +++++++++++++--- .../user_space/internal_listener_registry.h | 26 ++++++++++++-- .../user_space/thread_local_registry.cc | 0 .../user_space/thread_local_registry.h | 35 +++++++++++++++++++ source/server/BUILD | 3 ++ source/server/connection_handler_impl.cc | 22 ++++++++++-- source/server/connection_handler_impl.h | 4 ++- source/server/server.cc | 5 +-- source/server/worker_impl.cc | 3 +- source/server/worker_impl.h | 8 +++-- 14 files changed, 175 insertions(+), 17 deletions(-) create mode 100644 source/extensions/io_socket/user_space/thread_local_registry.cc create mode 100644 source/extensions/io_socket/user_space/thread_local_registry.h diff --git a/envoy/network/listener.h b/envoy/network/listener.h index fac995eb3ed6f..3fa9e7649a842 100644 --- a/envoy/network/listener.h +++ b/envoy/network/listener.h @@ -483,6 +483,25 @@ class InternalListenerManager { using InternalListenerManagerOptRef = absl::optional>; +class LocalInternalListenerRegistry { +public: + virtual ~LocalInternalListenerRegistry() = default; + + virtual void setInternalListenerManager(InternalListenerManager& internal_listener_manager) PURE; + + virtual Network::InternalListenerManagerOptRef getInternalListenerManager() PURE; +}; + +// The singleton interface of LocalInternalListenerRegistry accessor. The prod implementation is +// thread local. +class InternalListenerRegistry { +public: + virtual ~InternalListenerRegistry() = default; + + // This + virtual LocalInternalListenerRegistry* getLocalRegistry() PURE; +}; + /** * Handles delivering datagrams to the correct worker. */ diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 895488cfe6708..b778067f3f9c8 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -14,6 +14,7 @@ envoy_cc_extension( srcs = ["config.h"], deps = [ ":client_connection_factory", + ":internal_listener_registry", ":io_handle_impl_lib", ], ) @@ -69,6 +70,7 @@ envoy_cc_library( ], deps = [ ":io_handle_impl_lib", + ":thread_local_registry", "//envoy/network:client_connection_factory", "//envoy/network:connection_interface", "//envoy/registry", @@ -86,6 +88,24 @@ envoy_cc_library( "internal_listener_registry.h", ], deps = [ - "//source/server:bootstrap_extension_config_interface", - ] + ":thread_local_registry", + "//envoy/server:bootstrap_extension_config_interface", + ], +) + +envoy_cc_library( + name = "thread_local_registry", + srcs = [ + "thread_local_registry.cc", + ], + hdrs = [ + "client_connection_factory.h", + "thread_local_registry.h", + ], + deps = [ + "//envoy/network:client_connection_factory", + "//envoy/network:listener_interface", + "//envoy/singleton:manager_interface", + "//envoy/thread_local:thread_local_interface", + ], ) diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index 7fb7fdee59bc7..e4fee53174b1b 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -13,6 +13,9 @@ namespace Extensions { namespace IoSocket { namespace UserSpace { +ThreadLocal::TypedSlot* + InternalClientConnectionFactory::registry_tls_slot_ = nullptr; + Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnection( Event::Dispatcher& dispatcher, Network::Address::InstanceConstSharedPtr address, Network::Address::InstanceConstSharedPtr source_address, diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/io_socket/user_space/client_connection_factory.h index d143955b18273..c539434c97be9 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.h +++ b/source/extensions/io_socket/user_space/client_connection_factory.h @@ -8,6 +8,7 @@ #include "envoy/network/connection.h" #include "source/common/common/logger.h" +#include "source/extensions/io_socket/user_space/thread_local_registry.h" namespace Envoy { @@ -26,6 +27,15 @@ class InternalClientConnectionFactory : public Network::ClientConnectionFactory, Network::Address::InstanceConstSharedPtr source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) override; + // The slot is owned by the internal listener registry extension. Once that extension is + // intialized, this slot is available. The ClientConnectionFactory has two potential user cases. + // 1. The per worker thread connection handler populates the per worker listener registry. + // 2. A envoy thread local cluster lookup the per thread internal listener by listener name. + // Since the population and the lookup is supposed to be executed in the same worker thread, + // neither need to hold a lock. + // TODO(lambdai): make it friend to only bootstrap extension. + static ThreadLocal::TypedSlot* + registry_tls_slot_; }; } // namespace UserSpace diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.cc b/source/extensions/io_socket/user_space/internal_listener_registry.cc index fbcc33db5b9e3..96fadad984b4a 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.cc +++ b/source/extensions/io_socket/user_space/internal_listener_registry.cc @@ -1,15 +1,37 @@ #include "source/extensions/io_socket/user_space/internal_listener_registry.h" +#include "source/extensions/io_socket/user_space/client_connection_factory.h" +#include "envoy/singleton/manager.h" + namespace Envoy { namespace Extensions { namespace Bootstrap { namespace InternalListener { -void InternalListenerExtension::onServerInitialized() {} +void InternalListenerExtension::onServerInitialized() { + + std::shared_ptr internal_listener_registry = + server_context_.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(internal_listener_registry), + []() { return std::make_shared(); }); + + ASSERT(internal_listener_registry->tls_slot_ == nullptr); + + internal_listener_registry->tls_slot_ = + ThreadLocal::TypedSlot::makeUnique( + server_context_.threadLocal()); + internal_listener_registry->tls_slot_->set([](Event::Dispatcher&) { + return std::make_shared(); + }); + ASSERT(Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ == + nullptr); + Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ = + internal_listener_registry->tls_slot_.get(); +} -Server::BootstrapExtensionPtr InternalListenerRegisteryFactory::createBootstrapExtension( - const Protobuf::Message&, Server::Configuration::ServerFactoryContext&) { - return nullptr; +Server::BootstrapExtensionPtr InternalListenerRegistryFactory::createBootstrapExtension( + const Protobuf::Message&, Server::Configuration::ServerFactoryContext& context) { + return std::make_unique(context); } } // namespace InternalListener diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.h b/source/extensions/io_socket/user_space/internal_listener_registry.h index 9fa6f9c2346d3..821dc2037fc53 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.h +++ b/source/extensions/io_socket/user_space/internal_listener_registry.h @@ -4,6 +4,7 @@ #include "envoy/registry/registry.h" #include "envoy/server/bootstrap_extension_config.h" +#include "source/extensions/io_socket/user_space/thread_local_registry.h" #include "source/common/singleton/threadsafe_singleton.h" #include "absl/container/flat_hash_map.h" @@ -13,15 +14,32 @@ namespace Extensions { namespace Bootstrap { namespace InternalListener { -class InternalListenerExtension : public Server::Configuration::BootstrapExtension { +class TlsInternalListenerRegistry : public Singleton::Instance, + public Network::InternalListenerRegistry { public: + Network::LocalInternalListenerRegistry* getLocalRegistry() override { + if (auto opt = tls_slot_->get(); opt.has_value()) { + return &opt.value().get(); + } + return nullptr; + } + + std::unique_ptr> + tls_slot_; +}; +class InternalListenerExtension : public Server::BootstrapExtension { +public: + explicit InternalListenerExtension(Server::Configuration::ServerFactoryContext& server_context) + : server_context_(server_context) {} ~InternalListenerExtension() override = default; // Server::Configuration::BootstrapExtension void onServerInitialized() override; + + Server::Configuration::ServerFactoryContext& server_context_; }; -class InternalListenerRegisteryFactory : public Server::Configuration::BootstrapExtensionFactory { +class InternalListenerRegistryFactory : public Server::Configuration::BootstrapExtensionFactory { public: // Server::Configuration::BootstrapExtensionFactory Server::BootstrapExtensionPtr @@ -30,9 +48,11 @@ class InternalListenerRegisteryFactory : public Server::Configuration::Bootstrap ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique(); } - std::string name() const override { return "envoy.bootstrap.internal_listener"; }; + std::string name() const override { return "envoy.bootstrap.internal_listener_registr"; }; }; +SINGLETON_MANAGER_REGISTRATION(internal_listener_registry); + } // namespace InternalListener } // namespace Bootstrap } // namespace Extensions diff --git a/source/extensions/io_socket/user_space/thread_local_registry.cc b/source/extensions/io_socket/user_space/thread_local_registry.cc new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/source/extensions/io_socket/user_space/thread_local_registry.h b/source/extensions/io_socket/user_space/thread_local_registry.h new file mode 100644 index 0000000000000..8d29cac0d03cc --- /dev/null +++ b/source/extensions/io_socket/user_space/thread_local_registry.h @@ -0,0 +1,35 @@ +#pragma once + +#include "envoy/thread_local/thread_local_object.h" +#include "envoy/network/listener.h" + +namespace Envoy { +namespace Extensions { +namespace InternalListener { + +class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, + public Network::LocalInternalListenerRegistry { +public: + ThreadLocalRegistryImpl() = default; + + void + setInternalListenerManager(Network::InternalListenerManager& internal_listener_manager) override { + + ASSERT(manager_ == nullptr); + manager_ = &internal_listener_manager; + } + + Network::InternalListenerManagerOptRef getInternalListenerManager() override { + if (manager_ == nullptr) { + return Network::InternalListenerManagerOptRef(); + } + return Network::InternalListenerManagerOptRef(*manager_); + } + +private: + // A thread unsafe internal listener manager. + Network::InternalListenerManager* manager_{nullptr}; +}; +} // namespace InternalListener +} // namespace Extensions +} // namespace Envoy diff --git a/source/server/BUILD b/source/server/BUILD index 38443e8245f8a..a9d5eec928842 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -68,6 +68,8 @@ envoy_cc_library( ":active_tcp_listener", ":active_udp_listener", ":connection_handler_impl", + "//envoy/network:client_connection_factory", + "//source/common/network:address_lib", ], ) @@ -90,6 +92,7 @@ envoy_cc_library( "//envoy/network:listener_interface", "//envoy/runtime:runtime_interface", "//envoy/server:listener_manager_interface", + "//envoy/singleton:manager_interface", "//envoy/stats:timespan_interface", "//source/common/common:linked_object", "//source/common/common:non_copyable", diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index 24789e5feb5aa..f570e647af42a 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -4,6 +4,7 @@ #include "envoy/event/dispatcher.h" #include "envoy/network/filter.h" +#include "envoy/network/client_connection_factory.h" #include "source/common/common/logger.h" #include "source/common/event/deferred_task.h" @@ -11,14 +12,18 @@ #include "source/common/runtime/runtime_features.h" #include "source/server/active_internal_listener.h" #include "source/server/active_tcp_listener.h" +#include "envoy/singleton/manager.h" +#include "source/common/network/address_impl.h" namespace Envoy { namespace Server { ConnectionHandlerImpl::ConnectionHandlerImpl(Event::Dispatcher& dispatcher, - absl::optional worker_index) + absl::optional worker_index, + Singleton::Manager& singleton_manager) : worker_index_(worker_index), dispatcher_(dispatcher), - per_handler_stat_prefix_(dispatcher.name() + "."), disable_listeners_(false) {} + per_handler_stat_prefix_(dispatcher.name() + "."), disable_listeners_(false), + singleton_manager_(singleton_manager) {} void ConnectionHandlerImpl::incNumConnections() { ++num_handler_connections_; } @@ -41,6 +46,19 @@ void ConnectionHandlerImpl::addListener(absl::optional overridden_list ActiveListenerDetails details; if (config.internalListenerConfig().has_value()) { + // Link this ConnectionHandlerImpl to the thread local. Ideally this should be done during the + // initialization of ConnectionHandlerImpl. We move it here because the thread local object is + // not yet ready. + std::shared_ptr internal_listener_registry = + singleton_manager_.getTyped( + // TODO: avoid hacking into the singleton var implmentation. + "internal_listener_registry_singleton"); + RELEASE_ASSERT(internal_listener_registry != nullptr, + "Failed to get internal listener registry"); + Network::LocalInternalListenerRegistry* local_registry = + internal_listener_registry->getLocalRegistry(); + RELEASE_ASSERT(local_registry != nullptr, "Failed to get local internal listener registry"); + local_registry->setInternalListenerManager(*this); if (overridden_listener.has_value()) { for (auto& listener : listeners_) { if (listener.second.listener_->listenerTag() == overridden_listener) { diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index 23a39659c55d1..a0cd53d52348f 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -39,7 +39,8 @@ class ConnectionHandlerImpl : public Network::TcpConnectionHandler, using ActiveInternalListenerOptRef = absl::optional>; - ConnectionHandlerImpl(Event::Dispatcher& dispatcher, absl::optional worker_index); + ConnectionHandlerImpl(Event::Dispatcher& dispatcher, absl::optional worker_index, + Singleton::Manager& singleton_manager); // Network::ConnectionHandler uint64_t numConnections() const override { return num_handler_connections_; } @@ -97,6 +98,7 @@ class ConnectionHandlerImpl : public Network::TcpConnectionHandler, std::atomic num_handler_connections_{}; bool disable_listeners_; UnitFloat listener_reject_fraction_{UnitFloat::min()}; + Singleton::Manager& singleton_manager_; }; } // namespace Server diff --git a/source/server/server.cc b/source/server/server.cc index c914699197e6b..5ac5ee76d62e8 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -82,8 +82,9 @@ InstanceImpl::InstanceImpl( watermark_factory)), dispatcher_(api_->allocateDispatcher("main_thread")), singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory())), - handler_(new ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), - listener_component_factory_(*this), worker_factory_(thread_local_, *api_, hooks), + handler_(new ConnectionHandlerImpl(*dispatcher_, absl::nullopt, *singleton_manager_)), + listener_component_factory_(*this), + worker_factory_(thread_local_, *api_, hooks, *singleton_manager_), access_log_manager_(options.fileFlushIntervalMsec(), *api_, *dispatcher_, access_log_lock, store), terminated_(false), diff --git a/source/server/worker_impl.cc b/source/server/worker_impl.cc index bbadce5ce79fd..30304f8b64687 100644 --- a/source/server/worker_impl.cc +++ b/source/server/worker_impl.cc @@ -18,7 +18,8 @@ WorkerPtr ProdWorkerFactory::createWorker(uint32_t index, OverloadManager& overl const std::string& worker_name) { Event::DispatcherPtr dispatcher( api_.allocateDispatcher(worker_name, overload_manager.scaledTimerFactory())); - auto conn_handler = std::make_unique(*dispatcher, index); + auto conn_handler = + std::make_unique(*dispatcher, index, singleton_manager_); dispatcher->registerInternalListenerManager(*conn_handler); return std::make_unique(tls_, hooks_, std::move(dispatcher), std::move(conn_handler), overload_manager, api_, stat_names_); diff --git a/source/server/worker_impl.h b/source/server/worker_impl.h index 87419667eed87..41051cd57de65 100644 --- a/source/server/worker_impl.h +++ b/source/server/worker_impl.h @@ -9,6 +9,7 @@ #include "envoy/server/listener_manager.h" #include "envoy/server/worker.h" #include "envoy/thread_local/thread_local.h" +#include "envoy/singleton/manager.h" #include "source/common/common/logger.h" #include "source/server/listener_hooks.h" @@ -28,8 +29,10 @@ struct WorkerStatNames { class ProdWorkerFactory : public WorkerFactory, Logger::Loggable { public: - ProdWorkerFactory(ThreadLocal::Instance& tls, Api::Api& api, ListenerHooks& hooks) - : tls_(tls), api_(api), stat_names_(api.rootScope().symbolTable()), hooks_(hooks) {} + ProdWorkerFactory(ThreadLocal::Instance& tls, Api::Api& api, ListenerHooks& hooks, + Singleton::Manager& singleton_manager) + : tls_(tls), api_(api), stat_names_(api.rootScope().symbolTable()), hooks_(hooks), + singleton_manager_(singleton_manager) {} // Server::WorkerFactory WorkerPtr createWorker(uint32_t index, OverloadManager& overload_manager, @@ -40,6 +43,7 @@ class ProdWorkerFactory : public WorkerFactory, Logger::Loggable Date: Tue, 21 Dec 2021 23:55:52 +0000 Subject: [PATCH 18/59] see failure on singleton: singleton should be only used in main thread Signed-off-by: Yuchen Dai --- envoy/event/dispatcher.h | 13 ------------- source/common/event/dispatcher_impl.h | 10 ---------- .../user_space/client_connection_factory.cc | 8 +++++++- source/server/worker_impl.cc | 1 - test/common/quic/BUILD | 1 + test/common/quic/active_quic_listener_test.cc | 3 ++- test/common/quic/envoy_quic_dispatcher_test.cc | 5 ++++- test/extensions/common/proxy_protocol/BUILD | 1 + .../proxy_protocol_regression_test.cc | 4 +++- .../listener/proxy_protocol/proxy_protocol_test.cc | 6 ++++-- .../user_space/client_connection_factory_test.cc | 6 ++++-- test/integration/BUILD | 1 + test/integration/fake_upstream.cc | 3 ++- test/integration/fake_upstream.h | 2 ++ test/mocks/event/mocks.h | 3 --- test/mocks/event/wrapped_dispatcher.h | 9 --------- test/server/BUILD | 1 + test/server/connection_handler_test.cc | 6 +++++- 18 files changed, 37 insertions(+), 46 deletions(-) diff --git a/envoy/event/dispatcher.h b/envoy/event/dispatcher.h index b2ba88a273ba9..86fd2855a1512 100644 --- a/envoy/event/dispatcher.h +++ b/envoy/event/dispatcher.h @@ -218,19 +218,6 @@ class Dispatcher : public DispatcherBase, public ScopeTracker { Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) PURE; - /** - * Register an internal listener manager for this dispatcher. - */ - virtual void - registerInternalListenerManager(Network::InternalListenerManager& internal_listener_manager) PURE; - - /** - * @brief Get the Internal Listener Manager object. - * - * @return the registered internal istener manager or nullopt. - */ - virtual Network::InternalListenerManagerOptRef getInternalListenerManager() PURE; - /** * @return Filesystem::WatcherPtr a filesystem watcher owned by the caller. */ diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index d8d0fc9a0ba07..0b9359d14e4a8 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -67,16 +67,6 @@ class DispatcherImpl : Logger::Loggable, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) override; - void registerInternalListenerManager( - Network::InternalListenerManager& internal_listener_manager) override { - ASSERT(!internal_listener_manager_.has_value()); - internal_listener_manager_ = internal_listener_manager; - } - - Network::InternalListenerManagerOptRef getInternalListenerManager() override { - return internal_listener_manager_; - } - FileEventPtr createFileEvent(os_fd_t fd, FileReadyCb cb, FileTriggerType trigger, uint32_t events) override; Filesystem::WatcherPtr createFilesystemWatcher() override; diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index e4fee53174b1b..6cac9664aa09c 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -31,8 +31,14 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec address), source_address, std::move(transport_socket), options); + if (registry_tls_slot_ == nullptr || !registry_tls_slot_->get().has_value()) { + ENVOY_LOG_MISC(debug, "server has not initialized internal listener registry, close the connection"); + io_handle_server->close(); + return client_conn; + } + // It's either in the main thread or the worker is not yet started. - auto internal_listener_manager = dispatcher.getInternalListenerManager(); + auto internal_listener_manager = registry_tls_slot_->get()->getInternalListenerManager(); if (!internal_listener_manager.has_value()) { io_handle_server->close(); return client_conn; diff --git a/source/server/worker_impl.cc b/source/server/worker_impl.cc index 30304f8b64687..b56a664adc5e9 100644 --- a/source/server/worker_impl.cc +++ b/source/server/worker_impl.cc @@ -20,7 +20,6 @@ WorkerPtr ProdWorkerFactory::createWorker(uint32_t index, OverloadManager& overl api_.allocateDispatcher(worker_name, overload_manager.scaledTimerFactory())); auto conn_handler = std::make_unique(*dispatcher, index, singleton_manager_); - dispatcher->registerInternalListenerManager(*conn_handler); return std::make_unique(tls_, hooks_, std::move(dispatcher), std::move(conn_handler), overload_manager, api_, stat_names_); } diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD index f549b51982aec..009634c1605d5 100644 --- a/test/common/quic/BUILD +++ b/test/common/quic/BUILD @@ -228,6 +228,7 @@ envoy_cc_test( "//source/common/quic:envoy_quic_dispatcher_lib", "//source/common/quic:envoy_quic_proof_source_lib", "//source/common/quic:envoy_quic_server_session_lib", + "//source/common/singleton:manager_impl_lib", "//source/extensions/quic/crypto_stream:envoy_quic_crypto_server_stream_lib", "//source/server:configuration_lib", "//test/mocks/event:event_mocks", diff --git a/test/common/quic/active_quic_listener_test.cc b/test/common/quic/active_quic_listener_test.cc index 45cddb815629d..adc7737353518 100644 --- a/test/common/quic/active_quic_listener_test.cc +++ b/test/common/quic/active_quic_listener_test.cc @@ -79,7 +79,7 @@ class ActiveQuicListenerTest : public testing::TestWithParamallocateDispatcher("test_thread")), clock_(*dispatcher_), local_address_(Network::Test::getCanonicalLoopbackAddress(version_)), - connection_handler_(*dispatcher_, absl::nullopt), + connection_handler_(*dispatcher_, absl::nullopt, singleton_manager_), quic_version_(quic::CurrentSupportedHttp3Versions()[0]), quic_stat_names_(listener_config_.listenerScope().symbolTable()) {} @@ -294,6 +294,7 @@ class ActiveQuicListenerTest : public testing::TestWithParam listener_config_; NiceMock udp_packet_writer_factory_; quic::QuicConfig quic_config_; + Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; Server::ConnectionHandlerImpl connection_handler_; std::unique_ptr quic_listener_; Network::ActiveUdpListenerFactoryPtr listener_factory_; diff --git a/test/common/quic/envoy_quic_dispatcher_test.cc b/test/common/quic/envoy_quic_dispatcher_test.cc index 3481ebc731fa8..c888853fe7df3 100644 --- a/test/common/quic/envoy_quic_dispatcher_test.cc +++ b/test/common/quic/envoy_quic_dispatcher_test.cc @@ -32,6 +32,7 @@ #include "test/common/quic/test_utils.h" #include "source/common/quic/envoy_quic_alarm_factory.h" #include "source/common/quic/envoy_quic_utils.h" +#include "source/common/singleton/manager_impl.h" #include "source/extensions/quic/crypto_stream/envoy_quic_crypto_server_stream.h" #include "source/server/configuration_impl.h" #include "gmock/gmock.h" @@ -70,7 +71,8 @@ class EnvoyQuicDispatcherTest : public testing::TestWithParamthreadFactory()), + connection_handler_(*dispatcher_, absl::nullopt, singleton_manager_), envoy_quic_dispatcher_( &crypto_config_, quic_config_, &version_manager_, std::make_unique(*dispatcher_), @@ -234,6 +236,7 @@ class EnvoyQuicDispatcherTest : public testing::TestWithParamallocateDispatcher("test_thread")), socket_(std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()))), - connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), + connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt, singleton_manager_)), name_("proxy"), filter_chain_(Network::Test::createEmptyFilterChainWithRawBufferSockets()), init_manager_(nullptr) { EXPECT_CALL(socket_factory_, socketType()).WillOnce(Return(Network::Socket::Type::Stream)); @@ -168,6 +169,7 @@ class ProxyProtocolRegressionTest : public testing::TestWithParam socket_; Network::MockListenSocketFactory socket_factory_; Network::NopConnectionBalancerImpl connection_balancer_; + Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; Network::ConnectionHandlerPtr connection_handler_; Network::MockFilterChainFactory factory_; Network::ClientConnectionPtr conn_; diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc index 7fb371218a3d3..b8ed12de545ed 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc @@ -57,7 +57,7 @@ class ProxyProtocolTest : public testing::TestWithParamallocateDispatcher("test_thread")), socket_(std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()))), - connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), + connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt, singleton_manager_)), name_("proxy"), filter_chain_(Network::Test::createEmptyFilterChainWithRawBufferSockets()), init_manager_(nullptr) { EXPECT_CALL(socket_factory_, socketType()).WillOnce(Return(Network::Socket::Type::Stream)); @@ -194,6 +194,7 @@ class ProxyProtocolTest : public testing::TestWithParam socket_; Network::MockListenSocketFactory socket_factory_; Network::NopConnectionBalancerImpl connection_balancer_; + Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; Network::ConnectionHandlerPtr connection_handler_; Network::MockFilterChainFactory factory_; Network::ClientConnectionPtr conn_; @@ -1319,7 +1320,7 @@ class WildcardProxyProtocolTest : public testing::TestWithParamconnectionInfoProvider().localAddress()->ip()->port())), - connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), + connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt, singleton_manager_)), name_("proxy"), filter_chain_(Network::Test::createEmptyFilterChainWithRawBufferSockets()), init_manager_(nullptr) { EXPECT_CALL(socket_factory_, socketType()).WillOnce(Return(Network::Socket::Type::Stream)); @@ -1429,6 +1430,7 @@ class WildcardProxyProtocolTest : public testing::TestWithParam socket_; Network::Address::InstanceConstSharedPtr local_dst_address_; Network::NopConnectionBalancerImpl connection_balancer_; + Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; Network::ConnectionHandlerPtr connection_handler_; Network::MockFilterChainFactory factory_; Network::ClientConnectionPtr conn_; diff --git a/test/extensions/io_socket/user_space/client_connection_factory_test.cc b/test/extensions/io_socket/user_space/client_connection_factory_test.cc index d9f321ab6360f..6200b21b45111 100644 --- a/test/extensions/io_socket/user_space/client_connection_factory_test.cc +++ b/test/extensions/io_socket/user_space/client_connection_factory_test.cc @@ -71,7 +71,8 @@ TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalConnectionManagerNotEx TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalListenerNotExist) { MockInternalListenerManger internal_listener_manager; - dispatcher_->registerInternalListenerManager(internal_listener_manager); + // TODO(lambdai): fix attach the dispatcher to the thread local registry. + // dispatcher_->registerInternalListenerManager(internal_listener_manager); EXPECT_CALL(internal_listener_manager, findByAddress(_)) .WillOnce(testing::Return(Network::InternalListenerOptRef())); @@ -92,7 +93,8 @@ TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalListenerNotExist) { // does not instantiate a server connection. The server connection is tested in internal listener. TEST_F(ClientConnectionFactoryTest, ConnectSucceeds) { MockInternalListenerManger internal_listener_manager; - dispatcher_->registerInternalListenerManager(internal_listener_manager); + // TODO(lambdai): fix attach the dispatcher to the thread local registry. + // dispatcher_->registerInternalListenerManager(internal_listener_manager); MockInternalListener internal_listener; Network::InternalListenerOptRef internal_listener_opt{internal_listener}; diff --git a/test/integration/BUILD b/test/integration/BUILD index 23d783d586501..c1c4a039a0a9e 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -724,6 +724,7 @@ envoy_cc_test_library( "//source/common/config:utility_lib", "//source/common/grpc:codec_lib", "//source/common/grpc:common_lib", + "//source/common/singleton:manager_impl_lib", "//source/common/http/http1:codec_lib", "//source/common/http/http2:codec_lib", "//source/common/network:connection_balancer_lib", diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 38558114b6845..2f32cc51a0f37 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -531,7 +531,8 @@ FakeUpstream::FakeUpstream(Network::TransportSocketFactoryPtr&& transport_socket socket_factory_(std::make_unique(socket_)), api_(Api::createApiForTest(stats_store_)), time_system_(config.time_system_), dispatcher_(api_->allocateDispatcher("fake_upstream")), - handler_(new Server::ConnectionHandlerImpl(*dispatcher_, 0)), config_(config), + singleton_manager_(api_->threadFactory()), + handler_(new Server::ConnectionHandlerImpl(*dispatcher_, 0, singleton_manager_)), config_(config), read_disable_on_new_connection_(true), enable_half_close_(config.enable_half_close_), listener_(*this, http_type_ == Http::CodecType::HTTP3), filter_chain_(Network::Test::createEmptyFilterChain(std::move(transport_socket_factory))), diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index acb76221012a5..77954eff788ad 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -21,6 +21,7 @@ #include "source/common/common/basic_resource_impl.h" #include "source/common/common/callback_impl.h" #include "source/common/common/linked_object.h" +#include "source/common/singleton/manager_impl.h" #include "source/common/common/lock_guard.h" #include "source/common/common/thread.h" #include "source/common/config/utility.h" @@ -844,6 +845,7 @@ class FakeUpstream : Logger::Loggable, Api::ApiPtr api_; Event::TestTimeSystem& time_system_; Event::DispatcherPtr dispatcher_; + Singleton::ManagerImpl singleton_manager_; Network::ConnectionHandlerPtr handler_; std::list new_connections_ ABSL_GUARDED_BY(lock_); diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index cb86c15265c46..a155a75e75d10 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -60,9 +60,6 @@ class MockDispatcher : public Dispatcher { createClientConnection_(address, source_address, transport_socket, options)}; } - MOCK_METHOD(Network::InternalListenerManagerOptRef, getInternalListenerManager, ()); - MOCK_METHOD(void, registerInternalListenerManager, (Network::InternalListenerManager&)); - FileEventPtr createFileEvent(os_fd_t fd, FileReadyCb cb, FileTriggerType trigger, uint32_t events) override { return FileEventPtr{createFileEvent_(fd, cb, trigger, events)}; diff --git a/test/mocks/event/wrapped_dispatcher.h b/test/mocks/event/wrapped_dispatcher.h index 0d990b725add2..634235cbbd632 100644 --- a/test/mocks/event/wrapped_dispatcher.h +++ b/test/mocks/event/wrapped_dispatcher.h @@ -50,15 +50,6 @@ class WrappedDispatcher : public Dispatcher { std::move(transport_socket), options); } - void registerInternalListenerManager( - Network::InternalListenerManager& internal_listener_manager) override { - impl_.registerInternalListenerManager(internal_listener_manager); - } - - Network::InternalListenerManagerOptRef getInternalListenerManager() override { - return impl_.getInternalListenerManager(); - } - FileEventPtr createFileEvent(os_fd_t fd, FileReadyCb cb, FileTriggerType trigger, uint32_t events) override { return impl_.createFileEvent(fd, cb, trigger, events); diff --git a/test/server/BUILD b/test/server/BUILD index 3060862c1ef70..75ca1675c7899 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -80,6 +80,7 @@ envoy_cc_test( "//source/common/network:connection_balancer_lib", "//source/common/network:udp_packet_writer_handler_lib", "//source/common/stats:stats_lib", + "//source/common/singleton:manager_impl_lib", "//source/server:active_raw_udp_listener_config", "//source/server:connection_handler_lib", "//test/mocks/access_log:access_log_mocks", diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index d2b733072283b..a08aae985b18b 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -12,6 +12,8 @@ #include "source/common/network/address_impl.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/network/io_socket_handle_impl.h" +#include "source/common/api/api_impl.h" +#include "source/common/singleton/manager_impl.h" #include "source/common/network/raw_buffer_socket.h" #include "source/common/network/udp_listener_impl.h" #include "source/common/network/udp_packet_writer_handler_impl.h" @@ -48,7 +50,7 @@ namespace { class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable { public: ConnectionHandlerTest() - : handler_(new ConnectionHandlerImpl(dispatcher_, 0)), + : handler_(new ConnectionHandlerImpl(dispatcher_, 0, singleton_manager_)), filter_chain_(std::make_shared>()), listener_filter_matcher_(std::make_shared>()), access_log_(std::make_shared()) { @@ -333,6 +335,8 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable dispatcher_{"test"}; std::list listeners_; + Api::ApiPtr api_{Api::createApiForTest()}; + Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; std::unique_ptr handler_; NiceMock manager_; NiceMock factory_; From 7bbc65020d15027a7fd1a8a101e927b4a25ab3c3 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 6 Jan 2022 16:33:32 +0000 Subject: [PATCH 19/59] produce and save InternalListenerRegistry in extension factory Signed-off-by: Yuchen Dai --- envoy/network/listener.h | 3 ++ source/extensions/io_socket/user_space/BUILD | 2 ++ .../user_space/internal_listener_registry.cc | 25 ++++++++----- .../user_space/internal_listener_registry.h | 13 ++++--- .../user_space/thread_local_registry.h | 2 -- source/server/connection_handler_impl.cc | 16 ++++----- source/server/listener_impl.cc | 12 ++++++- source/server/listener_impl.h | 16 ++++++--- test/integration/BUILD | 2 ++ .../internal_listener_integration_test.cc | 5 +++ test/server/connection_handler_test.cc | 36 +++++++++++++++++-- test/server/listener_manager_impl_test.cc | 3 ++ test/server/listener_manager_impl_test.h | 21 +++++++++++ 13 files changed, 123 insertions(+), 33 deletions(-) diff --git a/envoy/network/listener.h b/envoy/network/listener.h index 3fa9e7649a842..f0404e48495f6 100644 --- a/envoy/network/listener.h +++ b/envoy/network/listener.h @@ -107,12 +107,15 @@ class UdpListenerConfig { using UdpListenerConfigOptRef = OptRef; +class InternalListenerRegistry; /** * Configuration for an internal listener. */ class InternalListenerConfig { public: virtual ~InternalListenerConfig() = default; + + virtual InternalListenerRegistry& internalListenerRegistry() PURE; }; using InternalListenerConfigOptRef = OptRef; diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index b778067f3f9c8..f81a277212d40 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -12,6 +12,8 @@ envoy_extension_package() envoy_cc_extension( name = "config", srcs = ["config.h"], + # for integration tests + visibility = ["//visibility:public"], deps = [ ":client_connection_factory", ":internal_listener_registry", diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.cc b/source/extensions/io_socket/user_space/internal_listener_registry.cc index 96fadad984b4a..b8a94e4fe3a6b 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.cc +++ b/source/extensions/io_socket/user_space/internal_listener_registry.cc @@ -2,31 +2,36 @@ #include "source/extensions/io_socket/user_space/client_connection_factory.h" #include "envoy/singleton/manager.h" +#include "source/common/singleton/threadsafe_singleton.h" namespace Envoy { namespace Extensions { namespace Bootstrap { namespace InternalListener { -void InternalListenerExtension::onServerInitialized() { +SINGLETON_MANAGER_REGISTRATION(internal_listener_registry); +void InternalListenerExtension::onServerInitialized() { + ENVOY_LOG_MISC(debug, "lambdai: InternalListenerExtension::onServerInitialized"); std::shared_ptr internal_listener_registry = server_context_.singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(internal_listener_registry), - []() { return std::make_shared(); }); - + [registry = tls_registry_]() { return registry; }); + // save it in the singleton + ASSERT(internal_listener_registry == tls_registry_); ASSERT(internal_listener_registry->tls_slot_ == nullptr); - - internal_listener_registry->tls_slot_ = + + tls_registry_->tls_slot_ = ThreadLocal::TypedSlot::makeUnique( server_context_.threadLocal()); - internal_listener_registry->tls_slot_->set([](Event::Dispatcher&) { + tls_registry_->tls_slot_->set([](Event::Dispatcher&) { return std::make_shared(); }); - ASSERT(Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ == - nullptr); + // This field is overwritten multiple times since onServerInitialized is called in each test case. + // ASSERT(Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ == + // nullptr); Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ = - internal_listener_registry->tls_slot_.get(); + tls_registry_->tls_slot_.get(); } Server::BootstrapExtensionPtr InternalListenerRegistryFactory::createBootstrapExtension( @@ -34,6 +39,8 @@ Server::BootstrapExtensionPtr InternalListenerRegistryFactory::createBootstrapEx return std::make_unique(context); } +REGISTER_FACTORY(InternalListenerRegistryFactory, Server::Configuration::BootstrapExtensionFactory); + } // namespace InternalListener } // namespace Bootstrap } // namespace Extensions diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.h b/source/extensions/io_socket/user_space/internal_listener_registry.h index 821dc2037fc53..754a084a3db55 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.h +++ b/source/extensions/io_socket/user_space/internal_listener_registry.h @@ -5,7 +5,6 @@ #include "envoy/server/bootstrap_extension_config.h" #include "source/extensions/io_socket/user_space/thread_local_registry.h" -#include "source/common/singleton/threadsafe_singleton.h" #include "absl/container/flat_hash_map.h" @@ -17,6 +16,10 @@ namespace InternalListener { class TlsInternalListenerRegistry : public Singleton::Instance, public Network::InternalListenerRegistry { public: + ~TlsInternalListenerRegistry() override { + ENVOY_LOG_MISC(debug, "lambdai: destroying TlsInternalListenerRegistry"); + } + Network::LocalInternalListenerRegistry* getLocalRegistry() override { if (auto opt = tls_slot_->get(); opt.has_value()) { return &opt.value().get(); @@ -30,13 +33,15 @@ class TlsInternalListenerRegistry : public Singleton::Instance, class InternalListenerExtension : public Server::BootstrapExtension { public: explicit InternalListenerExtension(Server::Configuration::ServerFactoryContext& server_context) - : server_context_(server_context) {} + : server_context_(server_context), + tls_registry_(std::make_shared()) {} ~InternalListenerExtension() override = default; // Server::Configuration::BootstrapExtension void onServerInitialized() override; Server::Configuration::ServerFactoryContext& server_context_; + std::shared_ptr tls_registry_; }; class InternalListenerRegistryFactory : public Server::Configuration::BootstrapExtensionFactory { @@ -48,11 +53,9 @@ class InternalListenerRegistryFactory : public Server::Configuration::BootstrapE ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique(); } - std::string name() const override { return "envoy.bootstrap.internal_listener_registr"; }; + std::string name() const override { return "envoy.bootstrap.internal_listener_registry"; }; }; -SINGLETON_MANAGER_REGISTRATION(internal_listener_registry); - } // namespace InternalListener } // namespace Bootstrap } // namespace Extensions diff --git a/source/extensions/io_socket/user_space/thread_local_registry.h b/source/extensions/io_socket/user_space/thread_local_registry.h index 8d29cac0d03cc..8de75ec403444 100644 --- a/source/extensions/io_socket/user_space/thread_local_registry.h +++ b/source/extensions/io_socket/user_space/thread_local_registry.h @@ -14,8 +14,6 @@ class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, void setInternalListenerManager(Network::InternalListenerManager& internal_listener_manager) override { - - ASSERT(manager_ == nullptr); manager_ = &internal_listener_manager; } diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index f570e647af42a..d321cd12e1d55 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -23,7 +23,9 @@ ConnectionHandlerImpl::ConnectionHandlerImpl(Event::Dispatcher& dispatcher, Singleton::Manager& singleton_manager) : worker_index_(worker_index), dispatcher_(dispatcher), per_handler_stat_prefix_(dispatcher.name() + "."), disable_listeners_(false), - singleton_manager_(singleton_manager) {} + singleton_manager_(singleton_manager) { + UNREFERENCED_PARAMETER(singleton_manager_); +} void ConnectionHandlerImpl::incNumConnections() { ++num_handler_connections_; } @@ -49,15 +51,11 @@ void ConnectionHandlerImpl::addListener(absl::optional overridden_list // Link this ConnectionHandlerImpl to the thread local. Ideally this should be done during the // initialization of ConnectionHandlerImpl. We move it here because the thread local object is // not yet ready. - std::shared_ptr internal_listener_registry = - singleton_manager_.getTyped( - // TODO: avoid hacking into the singleton var implmentation. - "internal_listener_registry_singleton"); - RELEASE_ASSERT(internal_listener_registry != nullptr, - "Failed to get internal listener registry"); + Network::InternalListenerRegistry& internal_listener_registry = + config.internalListenerConfig()->internalListenerRegistry(); Network::LocalInternalListenerRegistry* local_registry = - internal_listener_registry->getLocalRegistry(); - RELEASE_ASSERT(local_registry != nullptr, "Failed to get local internal listener registry"); + internal_listener_registry.getLocalRegistry(); + RELEASE_ASSERT(local_registry != nullptr, "Failed to get local internal listener registry."); local_registry->setInternalListenerManager(*this); if (overridden_listener.has_value()) { for (auto& listener : listeners_) { diff --git a/source/server/listener_impl.cc b/source/server/listener_impl.cc index e1d1788e9d161..2da17784681a9 100644 --- a/source/server/listener_impl.cc +++ b/source/server/listener_impl.cc @@ -9,6 +9,7 @@ #include "envoy/server/options.h" #include "envoy/server/transport_socket_config.h" #include "envoy/stats/scope.h" +#include "envoy/singleton/manager.h" #include "source/common/access_log/access_log_impl.h" #include "source/common/api/os_sys_calls_impl.h" @@ -468,7 +469,16 @@ void ListenerImpl::buildAccessLog() { void ListenerImpl::buildInternalListener() { if (config_.address().has_envoy_internal_address()) { - internal_listener_config_ = std::make_unique(); + std::shared_ptr internal_listener_registry = + parent_.server_.singletonManager().getTyped( + "internal_listener_registry_singleton"); + if (internal_listener_registry == nullptr) { + throw EnvoyException( + fmt::format("error adding listener '{}': internal listener registry is not initialized.", + address_->asString())); + } + internal_listener_config_ = + std::make_unique(*internal_listener_registry); if (config_.has_api_listener()) { throw EnvoyException( fmt::format("error adding listener '{}': internal address cannot be used in api listener", diff --git a/source/server/listener_impl.h b/source/server/listener_impl.h index 765631127c8e9..5b99ce61de8a9 100644 --- a/source/server/listener_impl.h +++ b/source/server/listener_impl.h @@ -377,11 +377,17 @@ class ListenerImpl final : public Network::ListenerConfig, Network::UdpListenerWorkerRouterPtr listener_worker_router_; }; - struct InternalListenerConfigImpl : public Network::InternalListenerConfig { - InternalListenerConfigImpl( - const envoy::config::listener::v3::Listener_InternalListenerConfig config) - : config_(config) {} - const envoy::config::listener::v3::Listener_InternalListenerConfig config_; + class InternalListenerConfigImpl : public Network::InternalListenerConfig { + public: + InternalListenerConfigImpl(Network::InternalListenerRegistry& internal_listener_registry) + : internal_listener_registry_(internal_listener_registry) {} + + Network::InternalListenerRegistry& internalListenerRegistry() override { + return internal_listener_registry_; + } + + private: + Network::InternalListenerRegistry& internal_listener_registry_; }; /** diff --git a/test/integration/BUILD b/test/integration/BUILD index c1c4a039a0a9e..95ac49a852257 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1656,6 +1656,8 @@ envoy_cc_test( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + # TODO: move to extenstion + "//source/extensions/io_socket/user_space:config", ], ) diff --git a/test/integration/internal_listener_integration_test.cc b/test/integration/internal_listener_integration_test.cc index 11614b84874a9..38bad718564b2 100644 --- a/test/integration/internal_listener_integration_test.cc +++ b/test/integration/internal_listener_integration_test.cc @@ -32,6 +32,11 @@ class InternalListenerIntegrationTest : public testing::TestWithParammutable_envoy_internal_address()->set_server_listener_name( "internal_listener"); }); + config_helper_.addBootstrapExtension(R"EOF( +name: envoy.bootstrap.internal_listener_registry +typed_config: + "@type": "type.googleapis.com/google.protobuf.Struct" +)EOF"); BaseIntegrationTest::initialize(); } }; diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index a08aae985b18b..541a5dc48382f 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -110,7 +110,38 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggableinternal_listener_config_ = - std::make_unique(); + std::make_unique(mock_listener_registry_); return listeners_.back().get(); } @@ -334,6 +365,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable dispatcher_{"test"}; + TestListener::MockInternalListenerRegistery mock_listener_registry_; std::list listeners_; Api::ApiPtr api_{Api::createApiForTest()}; Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 8002094c46467..02f1cb6c86093 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -48,6 +48,9 @@ using testing::Return; using testing::ReturnRef; using testing::Throw; +// For test only. +SINGLETON_MANAGER_REGISTRATION(internal_listener_registry); + class ListenerManagerImplWithDispatcherStatsTest : public ListenerManagerImplTest { protected: ListenerManagerImplWithDispatcherStatsTest() { enable_dispatcher_stats_ = true; } diff --git a/test/server/listener_manager_impl_test.h b/test/server/listener_manager_impl_test.h index 6febd935efa37..04da18f887c73 100644 --- a/test/server/listener_manager_impl_test.h +++ b/test/server/listener_manager_impl_test.h @@ -67,6 +67,14 @@ class ListenerManagerImplTest : public testing::Test { ListenerManagerImplTest() : api_(Api::createApiForTest(server_.api_.random_)) {} void SetUp() override { + // place holder. The + ASSERT_NE(nullptr, + server_.singletonManager().getTyped( + "internal_listener_registry_singleton", [registry = internal_registry_]() { + ENVOY_LOG_MISC(debug, "lambdai: create Dumb internal registry"); + return registry; + })); + ON_CALL(server_, api()).WillByDefault(ReturnRef(*api_)); EXPECT_CALL(worker_factory_, createWorker_()).WillOnce(Return(worker_)); ON_CALL(server_.validation_context_, staticValidationVisitor()) @@ -306,6 +314,19 @@ class ListenerManagerImplTest : public testing::Test { TestThreadsafeSingletonInjector os_calls_{&os_sys_calls_}; Api::OsSysCallsImpl os_sys_calls_actual_; NiceMock server_; + class DumbInternalListenerRegistry : public Singleton::Instance, + public Network::InternalListenerRegistry { + public: + ~DumbInternalListenerRegistry() { + ENVOY_LOG_MISC(debug, "lambdai: destroy DumbInternalListenerRegistry"); + } + MOCK_METHOD(Network::LocalInternalListenerRegistry*, getLocalRegistry, ()); + }; + std::shared_ptr internal_registry_{ + std::make_shared()}; + // TestThreadsafeSingletonInjector + // registry_{&internal_registry_}; + NiceMock listener_factory_; NiceMock validation_visitor; MockWorker* worker_ = new MockWorker(); From c987df43a9bf02fca8eaafeb698adc9c7b0c7501 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 6 Jan 2022 17:34:45 +0000 Subject: [PATCH 20/59] bind tls_registry in ClientConnectionFactoryTest Signed-off-by: Yuchen Dai --- test/extensions/io_socket/user_space/BUILD | 1 + .../client_connection_factory_test.cc | 39 ++++++++++++------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/test/extensions/io_socket/user_space/BUILD b/test/extensions/io_socket/user_space/BUILD index 86d68e0988c4a..b5cc3df4bc83d 100644 --- a/test/extensions/io_socket/user_space/BUILD +++ b/test/extensions/io_socket/user_space/BUILD @@ -69,5 +69,6 @@ envoy_extension_cc_test( "//source/extensions/io_socket/user_space:client_connection_factory", "//source/extensions/io_socket/user_space:io_handle_impl_lib", "//test/mocks/event:event_mocks", + "//test/mocks/thread_local:thread_local_mocks", ], ) diff --git a/test/extensions/io_socket/user_space/client_connection_factory_test.cc b/test/extensions/io_socket/user_space/client_connection_factory_test.cc index 6200b21b45111..53d33fd97886e 100644 --- a/test/extensions/io_socket/user_space/client_connection_factory_test.cc +++ b/test/extensions/io_socket/user_space/client_connection_factory_test.cc @@ -8,6 +8,7 @@ #include "source/extensions/io_socket/user_space/client_connection_factory.h" #include "source/extensions/io_socket/user_space/io_handle_impl.h" +#include "test/mocks/thread_local/mocks.h" #include "test/mocks/event/mocks.h" #include "test/test_common/network_utility.h" @@ -24,6 +25,12 @@ namespace UserSpace { namespace { +class MockInternalListenerManger : public Network::InternalListenerManager { +public: + MOCK_METHOD(Network::InternalListenerOptRef, findByAddress, + (const Network::Address::InstanceConstSharedPtr&)); +}; + // The internal connection factory is linked in this test suite. This test suite verifies the // connection can be created. class ClientConnectionFactoryTest : public testing::Test { @@ -32,10 +39,24 @@ class ClientConnectionFactoryTest : public testing::Test { : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")), buf_(1024) { std::tie(io_handle_, io_handle_peer_) = IoHandleFactory::createIoHandlePair(); + EXPECT_CALL(tls_allocator_, allocateSlot()); + tls_slot_ = + ThreadLocal::TypedSlot::makeUnique( + tls_allocator_); + tls_slot_->set([r = registry_](Event::Dispatcher&) { return r; }); + // TODO: restore the original value via RAII. + Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ = + tls_slot_.get(); } Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; + MockInternalListenerManger internal_listener_manager_; + std::shared_ptr registry_{ + std::make_shared()}; + ThreadLocal::MockInstance tls_allocator_; + std::unique_ptr> + tls_slot_; // Owned by IoHandleImpl. NiceMock* schedulable_cb_; @@ -51,11 +72,7 @@ class MockInternalListener : public Network::InternalListener { MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); }; -class MockInternalListenerManger : public Network::InternalListenerManager { -public: - MOCK_METHOD(Network::InternalListenerOptRef, findByAddress, - (const Network::Address::InstanceConstSharedPtr&)); -}; + TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalConnectionManagerNotExist) { auto client_conn = dispatcher_->createClientConnection( @@ -70,11 +87,9 @@ TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalConnectionManagerNotEx } TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalListenerNotExist) { - MockInternalListenerManger internal_listener_manager; - // TODO(lambdai): fix attach the dispatcher to the thread local registry. - // dispatcher_->registerInternalListenerManager(internal_listener_manager); + registry_->setInternalListenerManager(internal_listener_manager_); - EXPECT_CALL(internal_listener_manager, findByAddress(_)) + EXPECT_CALL(internal_listener_manager_, findByAddress(_)) .WillOnce(testing::Return(Network::InternalListenerOptRef())); auto client_conn = dispatcher_->createClientConnection( @@ -92,13 +107,11 @@ TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalListenerNotExist) { // Verify that the client connection to envoy internal address can be established. This test case // does not instantiate a server connection. The server connection is tested in internal listener. TEST_F(ClientConnectionFactoryTest, ConnectSucceeds) { - MockInternalListenerManger internal_listener_manager; - // TODO(lambdai): fix attach the dispatcher to the thread local registry. - // dispatcher_->registerInternalListenerManager(internal_listener_manager); + registry_->setInternalListenerManager(internal_listener_manager_); MockInternalListener internal_listener; Network::InternalListenerOptRef internal_listener_opt{internal_listener}; - EXPECT_CALL(internal_listener_manager, findByAddress(_)) + EXPECT_CALL(internal_listener_manager_, findByAddress(_)) .WillOnce(testing::Return(internal_listener_opt)); Network::ConnectionSocketPtr server_socket; EXPECT_CALL(internal_listener, onAccept(_)).WillOnce([&](auto&& socket) { From 98e80995f92b741de88770dfede30bbe5e9e7a07 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 6 Jan 2022 18:54:46 +0000 Subject: [PATCH 21/59] remove singleton dep from conn handler Signed-off-by: Yuchen Dai --- source/server/connection_handler_impl.cc | 8 ++------ source/server/connection_handler_impl.h | 4 +--- source/server/listener_impl.cc | 20 +++++++++---------- source/server/server.cc | 4 ++-- source/server/worker_impl.cc | 2 +- source/server/worker_impl.h | 7 ++----- test/common/quic/active_quic_listener_test.cc | 3 +-- .../common/quic/envoy_quic_dispatcher_test.cc | 4 +--- .../proxy_protocol_regression_test.cc | 4 ++-- .../proxy_protocol/proxy_protocol_test.cc | 6 ++---- test/integration/fake_upstream.cc | 3 +-- test/integration/fake_upstream.h | 1 - test/server/connection_handler_test.cc | 4 +--- test/server/listener_manager_impl_test.cc | 10 ++++++++++ test/server/listener_manager_impl_test.h | 10 ---------- 15 files changed, 36 insertions(+), 54 deletions(-) diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index d321cd12e1d55..ddf4385a30e9d 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -19,13 +19,9 @@ namespace Envoy { namespace Server { ConnectionHandlerImpl::ConnectionHandlerImpl(Event::Dispatcher& dispatcher, - absl::optional worker_index, - Singleton::Manager& singleton_manager) + absl::optional worker_index) : worker_index_(worker_index), dispatcher_(dispatcher), - per_handler_stat_prefix_(dispatcher.name() + "."), disable_listeners_(false), - singleton_manager_(singleton_manager) { - UNREFERENCED_PARAMETER(singleton_manager_); -} + per_handler_stat_prefix_(dispatcher.name() + "."), disable_listeners_(false) {} void ConnectionHandlerImpl::incNumConnections() { ++num_handler_connections_; } diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index a0cd53d52348f..23a39659c55d1 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -39,8 +39,7 @@ class ConnectionHandlerImpl : public Network::TcpConnectionHandler, using ActiveInternalListenerOptRef = absl::optional>; - ConnectionHandlerImpl(Event::Dispatcher& dispatcher, absl::optional worker_index, - Singleton::Manager& singleton_manager); + ConnectionHandlerImpl(Event::Dispatcher& dispatcher, absl::optional worker_index); // Network::ConnectionHandler uint64_t numConnections() const override { return num_handler_connections_; } @@ -98,7 +97,6 @@ class ConnectionHandlerImpl : public Network::TcpConnectionHandler, std::atomic num_handler_connections_{}; bool disable_listeners_; UnitFloat listener_reject_fraction_{UnitFloat::min()}; - Singleton::Manager& singleton_manager_; }; } // namespace Server diff --git a/source/server/listener_impl.cc b/source/server/listener_impl.cc index 2da17784681a9..70b1902f5a627 100644 --- a/source/server/listener_impl.cc +++ b/source/server/listener_impl.cc @@ -469,16 +469,6 @@ void ListenerImpl::buildAccessLog() { void ListenerImpl::buildInternalListener() { if (config_.address().has_envoy_internal_address()) { - std::shared_ptr internal_listener_registry = - parent_.server_.singletonManager().getTyped( - "internal_listener_registry_singleton"); - if (internal_listener_registry == nullptr) { - throw EnvoyException( - fmt::format("error adding listener '{}': internal listener registry is not initialized.", - address_->asString())); - } - internal_listener_config_ = - std::make_unique(*internal_listener_registry); if (config_.has_api_listener()) { throw EnvoyException( fmt::format("error adding listener '{}': internal address cannot be used in api listener", @@ -499,6 +489,16 @@ void ListenerImpl::buildInternalListener() { throw EnvoyException(fmt::format("error adding listener '{}': does not support socket option", address_->asString())); } + std::shared_ptr internal_listener_registry = + parent_.server_.singletonManager().getTyped( + "internal_listener_registry_singleton"); + if (internal_listener_registry == nullptr) { + throw EnvoyException( + fmt::format("error adding listener '{}': internal listener registry is not initialized.", + address_->asString())); + } + internal_listener_config_ = + std::make_unique(*internal_listener_registry); } else { if (config_.has_internal_listener()) { throw EnvoyException(fmt::format("error adding listener '{}': address is not an internal " diff --git a/source/server/server.cc b/source/server/server.cc index 5ac5ee76d62e8..701094c86a979 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -82,9 +82,9 @@ InstanceImpl::InstanceImpl( watermark_factory)), dispatcher_(api_->allocateDispatcher("main_thread")), singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory())), - handler_(new ConnectionHandlerImpl(*dispatcher_, absl::nullopt, *singleton_manager_)), + handler_(new ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), listener_component_factory_(*this), - worker_factory_(thread_local_, *api_, hooks, *singleton_manager_), + worker_factory_(thread_local_, *api_, hooks), access_log_manager_(options.fileFlushIntervalMsec(), *api_, *dispatcher_, access_log_lock, store), terminated_(false), diff --git a/source/server/worker_impl.cc b/source/server/worker_impl.cc index b56a664adc5e9..b89db29a11906 100644 --- a/source/server/worker_impl.cc +++ b/source/server/worker_impl.cc @@ -19,7 +19,7 @@ WorkerPtr ProdWorkerFactory::createWorker(uint32_t index, OverloadManager& overl Event::DispatcherPtr dispatcher( api_.allocateDispatcher(worker_name, overload_manager.scaledTimerFactory())); auto conn_handler = - std::make_unique(*dispatcher, index, singleton_manager_); + std::make_unique(*dispatcher, index); return std::make_unique(tls_, hooks_, std::move(dispatcher), std::move(conn_handler), overload_manager, api_, stat_names_); } diff --git a/source/server/worker_impl.h b/source/server/worker_impl.h index 41051cd57de65..f96aba082705d 100644 --- a/source/server/worker_impl.h +++ b/source/server/worker_impl.h @@ -29,10 +29,8 @@ struct WorkerStatNames { class ProdWorkerFactory : public WorkerFactory, Logger::Loggable { public: - ProdWorkerFactory(ThreadLocal::Instance& tls, Api::Api& api, ListenerHooks& hooks, - Singleton::Manager& singleton_manager) - : tls_(tls), api_(api), stat_names_(api.rootScope().symbolTable()), hooks_(hooks), - singleton_manager_(singleton_manager) {} + ProdWorkerFactory(ThreadLocal::Instance& tls, Api::Api& api, ListenerHooks& hooks) + : tls_(tls), api_(api), stat_names_(api.rootScope().symbolTable()), hooks_(hooks) {} // Server::WorkerFactory WorkerPtr createWorker(uint32_t index, OverloadManager& overload_manager, @@ -43,7 +41,6 @@ class ProdWorkerFactory : public WorkerFactory, Logger::LoggableallocateDispatcher("test_thread")), clock_(*dispatcher_), local_address_(Network::Test::getCanonicalLoopbackAddress(version_)), - connection_handler_(*dispatcher_, absl::nullopt, singleton_manager_), + connection_handler_(*dispatcher_, absl::nullopt), quic_version_(quic::CurrentSupportedHttp3Versions()[0]), quic_stat_names_(listener_config_.listenerScope().symbolTable()) {} @@ -294,7 +294,6 @@ class ActiveQuicListenerTest : public testing::TestWithParam listener_config_; NiceMock udp_packet_writer_factory_; quic::QuicConfig quic_config_; - Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; Server::ConnectionHandlerImpl connection_handler_; std::unique_ptr quic_listener_; Network::ActiveUdpListenerFactoryPtr listener_factory_; diff --git a/test/common/quic/envoy_quic_dispatcher_test.cc b/test/common/quic/envoy_quic_dispatcher_test.cc index c888853fe7df3..d33629c28da6c 100644 --- a/test/common/quic/envoy_quic_dispatcher_test.cc +++ b/test/common/quic/envoy_quic_dispatcher_test.cc @@ -71,8 +71,7 @@ class EnvoyQuicDispatcherTest : public testing::TestWithParamthreadFactory()), - connection_handler_(*dispatcher_, absl::nullopt, singleton_manager_), + connection_handler_(*dispatcher_, absl::nullopt), envoy_quic_dispatcher_( &crypto_config_, quic_config_, &version_manager_, std::make_unique(*dispatcher_), @@ -236,7 +235,6 @@ class EnvoyQuicDispatcherTest : public testing::TestWithParamallocateDispatcher("test_thread")), socket_(std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()))), - connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt, singleton_manager_)), + connection_handler_( + new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), name_("proxy"), filter_chain_(Network::Test::createEmptyFilterChainWithRawBufferSockets()), init_manager_(nullptr) { EXPECT_CALL(socket_factory_, socketType()).WillOnce(Return(Network::Socket::Type::Stream)); @@ -169,7 +170,6 @@ class ProxyProtocolRegressionTest : public testing::TestWithParam socket_; Network::MockListenSocketFactory socket_factory_; Network::NopConnectionBalancerImpl connection_balancer_; - Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; Network::ConnectionHandlerPtr connection_handler_; Network::MockFilterChainFactory factory_; Network::ClientConnectionPtr conn_; diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc index b8ed12de545ed..7fb371218a3d3 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc @@ -57,7 +57,7 @@ class ProxyProtocolTest : public testing::TestWithParamallocateDispatcher("test_thread")), socket_(std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()))), - connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt, singleton_manager_)), + connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), name_("proxy"), filter_chain_(Network::Test::createEmptyFilterChainWithRawBufferSockets()), init_manager_(nullptr) { EXPECT_CALL(socket_factory_, socketType()).WillOnce(Return(Network::Socket::Type::Stream)); @@ -194,7 +194,6 @@ class ProxyProtocolTest : public testing::TestWithParam socket_; Network::MockListenSocketFactory socket_factory_; Network::NopConnectionBalancerImpl connection_balancer_; - Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; Network::ConnectionHandlerPtr connection_handler_; Network::MockFilterChainFactory factory_; Network::ClientConnectionPtr conn_; @@ -1320,7 +1319,7 @@ class WildcardProxyProtocolTest : public testing::TestWithParamconnectionInfoProvider().localAddress()->ip()->port())), - connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt, singleton_manager_)), + connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), name_("proxy"), filter_chain_(Network::Test::createEmptyFilterChainWithRawBufferSockets()), init_manager_(nullptr) { EXPECT_CALL(socket_factory_, socketType()).WillOnce(Return(Network::Socket::Type::Stream)); @@ -1430,7 +1429,6 @@ class WildcardProxyProtocolTest : public testing::TestWithParam socket_; Network::Address::InstanceConstSharedPtr local_dst_address_; Network::NopConnectionBalancerImpl connection_balancer_; - Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; Network::ConnectionHandlerPtr connection_handler_; Network::MockFilterChainFactory factory_; Network::ClientConnectionPtr conn_; diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 2f32cc51a0f37..38558114b6845 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -531,8 +531,7 @@ FakeUpstream::FakeUpstream(Network::TransportSocketFactoryPtr&& transport_socket socket_factory_(std::make_unique(socket_)), api_(Api::createApiForTest(stats_store_)), time_system_(config.time_system_), dispatcher_(api_->allocateDispatcher("fake_upstream")), - singleton_manager_(api_->threadFactory()), - handler_(new Server::ConnectionHandlerImpl(*dispatcher_, 0, singleton_manager_)), config_(config), + handler_(new Server::ConnectionHandlerImpl(*dispatcher_, 0)), config_(config), read_disable_on_new_connection_(true), enable_half_close_(config.enable_half_close_), listener_(*this, http_type_ == Http::CodecType::HTTP3), filter_chain_(Network::Test::createEmptyFilterChain(std::move(transport_socket_factory))), diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 77954eff788ad..f180638b33f83 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -845,7 +845,6 @@ class FakeUpstream : Logger::Loggable, Api::ApiPtr api_; Event::TestTimeSystem& time_system_; Event::DispatcherPtr dispatcher_; - Singleton::ManagerImpl singleton_manager_; Network::ConnectionHandlerPtr handler_; std::list new_connections_ ABSL_GUARDED_BY(lock_); diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 541a5dc48382f..cd9badcfabc98 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -50,7 +50,7 @@ namespace { class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable { public: ConnectionHandlerTest() - : handler_(new ConnectionHandlerImpl(dispatcher_, 0, singleton_manager_)), + : handler_(new ConnectionHandlerImpl(dispatcher_, 0)), filter_chain_(std::make_shared>()), listener_filter_matcher_(std::make_shared>()), access_log_(std::make_shared()) { @@ -367,8 +367,6 @@ public : NiceMock dispatcher_{"test"}; TestListener::MockInternalListenerRegistery mock_listener_registry_; std::list listeners_; - Api::ApiPtr api_{Api::createApiForTest()}; - Singleton::ManagerImpl singleton_manager_{api_->threadFactory()}; std::unique_ptr handler_; NiceMock manager_; NiceMock factory_; diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 02f1cb6c86093..6099f6735fff3 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -58,6 +58,16 @@ class ListenerManagerImplWithDispatcherStatsTest : public ListenerManagerImplTes class ListenerManagerImplWithRealFiltersTest : public ListenerManagerImplTest { public: + void SetUp() override { + ListenerManagerImplTest::SetUp(); + // place holder. The + ASSERT_NE(nullptr, + server_.singletonManager().getTyped( + "internal_listener_registry_singleton", [registry = internal_registry_]() { + ENVOY_LOG_MISC(debug, "lambdai: create Dumb internal registry"); + return registry; + })); + } /** * Create an IPv4 listener with a given name. */ diff --git a/test/server/listener_manager_impl_test.h b/test/server/listener_manager_impl_test.h index 04da18f887c73..969025bd83261 100644 --- a/test/server/listener_manager_impl_test.h +++ b/test/server/listener_manager_impl_test.h @@ -67,14 +67,6 @@ class ListenerManagerImplTest : public testing::Test { ListenerManagerImplTest() : api_(Api::createApiForTest(server_.api_.random_)) {} void SetUp() override { - // place holder. The - ASSERT_NE(nullptr, - server_.singletonManager().getTyped( - "internal_listener_registry_singleton", [registry = internal_registry_]() { - ENVOY_LOG_MISC(debug, "lambdai: create Dumb internal registry"); - return registry; - })); - ON_CALL(server_, api()).WillByDefault(ReturnRef(*api_)); EXPECT_CALL(worker_factory_, createWorker_()).WillOnce(Return(worker_)); ON_CALL(server_.validation_context_, staticValidationVisitor()) @@ -324,8 +316,6 @@ class ListenerManagerImplTest : public testing::Test { }; std::shared_ptr internal_registry_{ std::make_shared()}; - // TestThreadsafeSingletonInjector - // registry_{&internal_registry_}; NiceMock listener_factory_; NiceMock validation_visitor; From 476b44bd56a17ddb4369e58a4acb92d175e7c0f3 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Sat, 8 Jan 2022 01:13:25 +0000 Subject: [PATCH 22/59] better conn lib Signed-off-by: Yuchen Dai --- envoy/network/client_connection_factory.h | 11 ++++++++- envoy/network/listener.h | 18 ++++++++++---- source/common/event/BUILD | 2 +- source/common/event/dispatcher_impl.cc | 7 ++++-- source/common/event/dispatcher_impl.h | 2 -- source/common/network/BUILD | 18 +++++++------- .../user_space/client_connection_factory.cc | 5 ++-- .../user_space/internal_listener_registry.cc | 5 ++-- .../user_space/internal_listener_registry.h | 4 ---- .../user_space/thread_local_registry.h | 2 +- source/server/connection_handler_impl.cc | 11 ++++----- source/server/listener_impl.cc | 2 +- source/server/server.cc | 3 +-- source/server/worker_impl.cc | 3 +-- source/server/worker_impl.h | 2 +- .../proxy_protocol_regression_test.cc | 5 ++-- .../client_connection_factory_test.cc | 4 +--- test/integration/fake_upstream.h | 2 +- test/server/BUILD | 2 +- test/server/connection_handler_test.cc | 24 +++++++++---------- 20 files changed, 73 insertions(+), 59 deletions(-) diff --git a/envoy/network/client_connection_factory.h b/envoy/network/client_connection_factory.h index 142f4f54b7df7..defffd1f7685a 100644 --- a/envoy/network/client_connection_factory.h +++ b/envoy/network/client_connection_factory.h @@ -9,14 +9,23 @@ namespace Envoy { namespace Network { +// The factory to create a client connection. This factory hides the details of various remote +// address type and transport socket. class ClientConnectionFactory : public Config::UntypedFactory { public: - ClientConnectionFactory() = default; ~ClientConnectionFactory() override = default; // Config::UntypedFactory std::string category() const override { return "network.connection"; } + /** + * @param address The target remote address. + * @param source_address Optional source address. + * @param transport_socket The transport socket which supports this client connection. + * @param options The optional socket options. + * @return Network::ClientConnectionPtr The created connection. It's never nullptr but the return + * connection may be closed upon return. + */ virtual Network::ClientConnectionPtr createClientConnection(Event::Dispatcher& dispatcher, Network::Address::InstanceConstSharedPtr address, diff --git a/envoy/network/listener.h b/envoy/network/listener.h index f0404e48495f6..0771fdff3aed3 100644 --- a/envoy/network/listener.h +++ b/envoy/network/listener.h @@ -107,7 +107,9 @@ class UdpListenerConfig { using UdpListenerConfigOptRef = OptRef; +// Forward declare. class InternalListenerRegistry; + /** * Configuration for an internal listener. */ @@ -115,6 +117,10 @@ class InternalListenerConfig { public: virtual ~InternalListenerConfig() = default; + /** + * @return InternalListenerRegistry& The internal registry of this internal listener config. The + * registry outlives this listener config. + */ virtual InternalListenerRegistry& internalListenerRegistry() PURE; }; @@ -475,7 +481,7 @@ class InternalListenerManager { virtual ~InternalListenerManager() = default; /** - * Return the internal listener callbacks binding the listener address. + * Return the internal listene binding the listener address. * * @param listen_address the internal address of the expected internal listener. */ @@ -486,22 +492,26 @@ class InternalListenerManager { using InternalListenerManagerOptRef = absl::optional>; +// The thread local registry. class LocalInternalListenerRegistry { public: virtual ~LocalInternalListenerRegistry() = default; + // Set the internal listener manager which maintains life of internal listeners. Called by connection handler. virtual void setInternalListenerManager(InternalListenerManager& internal_listener_manager) PURE; + // Get the internal listener manager to obtain a listener. Called by client connection factory. virtual Network::InternalListenerManagerOptRef getInternalListenerManager() PURE; }; -// The singleton interface of LocalInternalListenerRegistry accessor. The prod implementation is -// thread local. +// The central internal listener registry interface providing the thread local accessor. class InternalListenerRegistry { public: virtual ~InternalListenerRegistry() = default; - // This + /** + * @return The thread local registry. + */ virtual LocalInternalListenerRegistry* getLocalRegistry() PURE; }; diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 8a1dafbe880e8..302319d88c808 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -47,7 +47,7 @@ envoy_cc_library( "//source/common/config:utility_lib", "//source/common/filesystem:watcher_lib", "//source/common/network:address_lib", - "//source/common/network:connection_lib", + "//source/common/network:default_client_connection_factory", "//source/common/network:listener_lib", "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", ], diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 2b45f13601586..72d8029c27af3 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -152,9 +152,12 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr const Network::ConnectionSocket::OptionsSharedPtr& options) { ASSERT(isThreadSafe()); - auto factory = Config::Utility::getFactoryByName( + auto* factory = Config::Utility::getFactoryByName( std::string(Network::Address::addressType(address))); - // The caller expects a non-null connection as of today. A unsupported address will crash any way. + // The target address is usually offered by EDS and the EDS api should reject the supported + // address. Alternatively, we can create a closed connection to avoid crash. Note that the caller + // expects a non-null connection as of today so we cannot gracefully handle unsupported address + // type. return factory->createClientConnection(*this, address, source_address, std::move(transport_socket), options); } diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index 0b9359d14e4a8..068638b727252 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -66,7 +66,6 @@ class DispatcherImpl : Logger::Loggable, Network::Address::InstanceConstSharedPtr source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) override; - FileEventPtr createFileEvent(os_fd_t fd, FileReadyCb cb, FileTriggerType trigger, uint32_t events) override; Filesystem::WatcherPtr createFilesystemWatcher() override; @@ -173,7 +172,6 @@ class DispatcherImpl : Logger::Loggable, MonotonicTime approximate_monotonic_time_; WatchdogRegistrationPtr watchdog_registration_; const ScaledRangeTimerManagerPtr scaled_timer_manager_; - Network::InternalListenerManagerOptRef internal_listener_manager_; }; } // namespace Event diff --git a/source/common/network/BUILD b/source/common/network/BUILD index dea4dbe2d3da1..4e0ee6aee762c 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -73,13 +73,20 @@ envoy_cc_library( envoy_cc_library( name = "connection_lib", + deps = [ + ":connection_impl", + ":default_client_connection_factory", + ], +) + +envoy_cc_library( + name = "connection_impl", srcs = ["connection_impl.cc"], hdrs = ["connection_impl.h"], external_deps = ["abseil_optional"], deps = [ ":address_lib", ":connection_base_lib", - ":default_client_connection_factory", ":raw_buffer_socket_lib", ":utility_lib", "//envoy/event:timer_interface", @@ -115,21 +122,16 @@ envoy_cc_library( "default_client_connection_factory.cc", ], hdrs = [ - "connection_impl.h", "default_client_connection_factory.h", ], deps = [ ":address_lib", + ":connection_base_lib", + ":connection_impl", "//envoy/network:client_connection_factory", "//envoy/network:connection_interface", "//envoy/network:transport_socket_interface", "//envoy/registry", - - # required by connection_impl.h - # will organize below when default client factory is confirmed to use dedicated target. - "//source/common/buffer:watermark_buffer_lib", - ":connection_base_lib", - "//source/common/stream_info:stream_info_lib", ], ) diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index 6cac9664aa09c..11a28cf0d2fa4 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -32,7 +32,8 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec source_address, std::move(transport_socket), options); if (registry_tls_slot_ == nullptr || !registry_tls_slot_->get().has_value()) { - ENVOY_LOG_MISC(debug, "server has not initialized internal listener registry, close the connection"); + ENVOY_LOG_MISC(debug, + "server has not initialized internal listener registry, close the connection"); io_handle_server->close(); return client_conn; } @@ -44,7 +45,7 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec return client_conn; } - // The request internal listener may not exist. + // The requested internal listener is not available yet. auto internal_listener = internal_listener_manager.value().get().findByAddress(address); if (!internal_listener.has_value()) { io_handle_server->close(); diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.cc b/source/extensions/io_socket/user_space/internal_listener_registry.cc index b8a94e4fe3a6b..b8b3d1a15b87b 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.cc +++ b/source/extensions/io_socket/user_space/internal_listener_registry.cc @@ -1,8 +1,9 @@ #include "source/extensions/io_socket/user_space/internal_listener_registry.h" -#include "source/extensions/io_socket/user_space/client_connection_factory.h" #include "envoy/singleton/manager.h" + #include "source/common/singleton/threadsafe_singleton.h" +#include "source/extensions/io_socket/user_space/client_connection_factory.h" namespace Envoy { namespace Extensions { @@ -20,7 +21,7 @@ void InternalListenerExtension::onServerInitialized() { // save it in the singleton ASSERT(internal_listener_registry == tls_registry_); ASSERT(internal_listener_registry->tls_slot_ == nullptr); - + tls_registry_->tls_slot_ = ThreadLocal::TypedSlot::makeUnique( server_context_.threadLocal()); diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.h b/source/extensions/io_socket/user_space/internal_listener_registry.h index 754a084a3db55..d1323944f8572 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.h +++ b/source/extensions/io_socket/user_space/internal_listener_registry.h @@ -16,10 +16,6 @@ namespace InternalListener { class TlsInternalListenerRegistry : public Singleton::Instance, public Network::InternalListenerRegistry { public: - ~TlsInternalListenerRegistry() override { - ENVOY_LOG_MISC(debug, "lambdai: destroying TlsInternalListenerRegistry"); - } - Network::LocalInternalListenerRegistry* getLocalRegistry() override { if (auto opt = tls_slot_->get(); opt.has_value()) { return &opt.value().get(); diff --git a/source/extensions/io_socket/user_space/thread_local_registry.h b/source/extensions/io_socket/user_space/thread_local_registry.h index 8de75ec403444..dd03ba464545a 100644 --- a/source/extensions/io_socket/user_space/thread_local_registry.h +++ b/source/extensions/io_socket/user_space/thread_local_registry.h @@ -1,7 +1,7 @@ #pragma once -#include "envoy/thread_local/thread_local_object.h" #include "envoy/network/listener.h" +#include "envoy/thread_local/thread_local_object.h" namespace Envoy { namespace Extensions { diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index ddf4385a30e9d..40aa8ae0bd3e2 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -3,17 +3,17 @@ #include #include "envoy/event/dispatcher.h" -#include "envoy/network/filter.h" #include "envoy/network/client_connection_factory.h" +#include "envoy/network/filter.h" +#include "envoy/singleton/manager.h" #include "source/common/common/logger.h" #include "source/common/event/deferred_task.h" +#include "source/common/network/address_impl.h" #include "source/common/network/utility.h" #include "source/common/runtime/runtime_features.h" #include "source/server/active_internal_listener.h" #include "source/server/active_tcp_listener.h" -#include "envoy/singleton/manager.h" -#include "source/common/network/address_impl.h" namespace Envoy { namespace Server { @@ -44,9 +44,8 @@ void ConnectionHandlerImpl::addListener(absl::optional overridden_list ActiveListenerDetails details; if (config.internalListenerConfig().has_value()) { - // Link this ConnectionHandlerImpl to the thread local. Ideally this should be done during the - // initialization of ConnectionHandlerImpl. We move it here because the thread local object is - // not yet ready. + // Ensure the this ConnectionHandlerImpl link to the thread local registry. Ideally this step + // should be done only once. However, an extra phase and interface is overkill. Network::InternalListenerRegistry& internal_listener_registry = config.internalListenerConfig()->internalListenerRegistry(); Network::LocalInternalListenerRegistry* local_registry = diff --git a/source/server/listener_impl.cc b/source/server/listener_impl.cc index 70b1902f5a627..3aa701368a975 100644 --- a/source/server/listener_impl.cc +++ b/source/server/listener_impl.cc @@ -8,8 +8,8 @@ #include "envoy/registry/registry.h" #include "envoy/server/options.h" #include "envoy/server/transport_socket_config.h" -#include "envoy/stats/scope.h" #include "envoy/singleton/manager.h" +#include "envoy/stats/scope.h" #include "source/common/access_log/access_log_impl.h" #include "source/common/api/os_sys_calls_impl.h" diff --git a/source/server/server.cc b/source/server/server.cc index 701094c86a979..c914699197e6b 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -83,8 +83,7 @@ InstanceImpl::InstanceImpl( dispatcher_(api_->allocateDispatcher("main_thread")), singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory())), handler_(new ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), - listener_component_factory_(*this), - worker_factory_(thread_local_, *api_, hooks), + listener_component_factory_(*this), worker_factory_(thread_local_, *api_, hooks), access_log_manager_(options.fileFlushIntervalMsec(), *api_, *dispatcher_, access_log_lock, store), terminated_(false), diff --git a/source/server/worker_impl.cc b/source/server/worker_impl.cc index b89db29a11906..3e0fdbc93e2bc 100644 --- a/source/server/worker_impl.cc +++ b/source/server/worker_impl.cc @@ -18,8 +18,7 @@ WorkerPtr ProdWorkerFactory::createWorker(uint32_t index, OverloadManager& overl const std::string& worker_name) { Event::DispatcherPtr dispatcher( api_.allocateDispatcher(worker_name, overload_manager.scaledTimerFactory())); - auto conn_handler = - std::make_unique(*dispatcher, index); + auto conn_handler = std::make_unique(*dispatcher, index); return std::make_unique(tls_, hooks_, std::move(dispatcher), std::move(conn_handler), overload_manager, api_, stat_names_); } diff --git a/source/server/worker_impl.h b/source/server/worker_impl.h index f96aba082705d..326f489ac2dd7 100644 --- a/source/server/worker_impl.h +++ b/source/server/worker_impl.h @@ -8,8 +8,8 @@ #include "envoy/server/guarddog.h" #include "envoy/server/listener_manager.h" #include "envoy/server/worker.h" -#include "envoy/thread_local/thread_local.h" #include "envoy/singleton/manager.h" +#include "envoy/thread_local/thread_local.h" #include "source/common/common/logger.h" #include "source/server/listener_hooks.h" diff --git a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc index 9b9acaae953c1..f00ea246a7601 100644 --- a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc +++ b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc @@ -5,10 +5,10 @@ #include "source/common/event/dispatcher_impl.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/network/listen_socket_impl.h" +#include "source/common/singleton/manager_impl.h" #include "source/extensions/common/proxy_protocol/proxy_protocol_header.h" #include "source/extensions/filters/listener/proxy_protocol/proxy_protocol.h" #include "source/server/connection_handler_impl.h" -#include "source/common/singleton/manager_impl.h" #include "test/mocks/buffer/mocks.h" #include "test/mocks/network/mocks.h" @@ -44,8 +44,7 @@ class ProxyProtocolRegressionTest : public testing::TestWithParamallocateDispatcher("test_thread")), socket_(std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()))), - connection_handler_( - new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), + connection_handler_(new Server::ConnectionHandlerImpl(*dispatcher_, absl::nullopt)), name_("proxy"), filter_chain_(Network::Test::createEmptyFilterChainWithRawBufferSockets()), init_manager_(nullptr) { EXPECT_CALL(socket_factory_, socketType()).WillOnce(Return(Network::Socket::Type::Stream)); diff --git a/test/extensions/io_socket/user_space/client_connection_factory_test.cc b/test/extensions/io_socket/user_space/client_connection_factory_test.cc index 53d33fd97886e..d5c41b021bba3 100644 --- a/test/extensions/io_socket/user_space/client_connection_factory_test.cc +++ b/test/extensions/io_socket/user_space/client_connection_factory_test.cc @@ -8,8 +8,8 @@ #include "source/extensions/io_socket/user_space/client_connection_factory.h" #include "source/extensions/io_socket/user_space/io_handle_impl.h" -#include "test/mocks/thread_local/mocks.h" #include "test/mocks/event/mocks.h" +#include "test/mocks/thread_local/mocks.h" #include "test/test_common/network_utility.h" #include "absl/container/fixed_array.h" @@ -72,8 +72,6 @@ class MockInternalListener : public Network::InternalListener { MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); }; - - TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalConnectionManagerNotExist) { auto client_conn = dispatcher_->createClientConnection( std::make_shared(listener_addr), diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index f180638b33f83..40574b085af72 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -21,7 +21,6 @@ #include "source/common/common/basic_resource_impl.h" #include "source/common/common/callback_impl.h" #include "source/common/common/linked_object.h" -#include "source/common/singleton/manager_impl.h" #include "source/common/common/lock_guard.h" #include "source/common/common/thread.h" #include "source/common/config/utility.h" @@ -35,6 +34,7 @@ #include "source/common/network/listen_socket_impl.h" #include "source/common/network/udp_listener_impl.h" #include "source/common/network/udp_packet_writer_handler_impl.h" +#include "source/common/singleton/manager_impl.h" #include "source/common/stats/isolated_store_impl.h" #if defined(ENVOY_ENABLE_QUIC) diff --git a/test/server/BUILD b/test/server/BUILD index 75ca1675c7899..e314d9436c4c1 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -79,8 +79,8 @@ envoy_cc_test( "//source/common/network:address_lib", "//source/common/network:connection_balancer_lib", "//source/common/network:udp_packet_writer_handler_lib", - "//source/common/stats:stats_lib", "//source/common/singleton:manager_impl_lib", + "//source/common/stats:stats_lib", "//source/server:active_raw_udp_listener_config", "//source/server:connection_handler_lib", "//test/mocks/access_log:access_log_mocks", diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index cd9badcfabc98..11cbef74226e5 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -7,17 +7,17 @@ #include "envoy/network/filter.h" #include "envoy/stats/scope.h" +#include "source/common/api/api_impl.h" #include "source/common/common/utility.h" #include "source/common/config/utility.h" #include "source/common/network/address_impl.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/network/io_socket_handle_impl.h" -#include "source/common/api/api_impl.h" -#include "source/common/singleton/manager_impl.h" #include "source/common/network/raw_buffer_socket.h" #include "source/common/network/udp_listener_impl.h" #include "source/common/network/udp_packet_writer_handler_impl.h" #include "source/common/network/utility.h" +#include "source/common/singleton/manager_impl.h" #include "source/server/active_raw_udp_listener_config.h" #include "source/server/connection_handler_impl.h" @@ -117,18 +117,18 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable Date: Mon, 10 Jan 2022 18:01:46 +0000 Subject: [PATCH 23/59] more clean up Signed-off-by: Yuchen Dai --- source/extensions/io_socket/user_space/BUILD | 4 ---- .../io_socket/user_space/internal_listener_registry.cc | 6 ++---- .../io_socket/user_space/thread_local_registry.h | 1 - source/server/BUILD | 1 - source/server/connection_handler_impl.cc | 2 -- source/server/worker_impl.h | 1 - test/common/quic/BUILD | 1 - test/common/quic/envoy_quic_dispatcher_test.cc | 1 - test/extensions/common/proxy_protocol/BUILD | 1 - .../proxy_protocol/proxy_protocol_regression_test.cc | 1 - test/integration/BUILD | 1 - test/integration/fake_upstream.h | 1 - test/server/BUILD | 1 - test/server/connection_handler_test.cc | 6 ++---- test/server/listener_manager_impl_test.cc | 10 +++------- test/server/listener_manager_impl_test.h | 3 --- 16 files changed, 7 insertions(+), 34 deletions(-) diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index f81a277212d40..9b9f1153b0dcf 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -97,9 +97,6 @@ envoy_cc_library( envoy_cc_library( name = "thread_local_registry", - srcs = [ - "thread_local_registry.cc", - ], hdrs = [ "client_connection_factory.h", "thread_local_registry.h", @@ -107,7 +104,6 @@ envoy_cc_library( deps = [ "//envoy/network:client_connection_factory", "//envoy/network:listener_interface", - "//envoy/singleton:manager_interface", "//envoy/thread_local:thread_local_interface", ], ) diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.cc b/source/extensions/io_socket/user_space/internal_listener_registry.cc index b8b3d1a15b87b..6dd85b46404b4 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.cc +++ b/source/extensions/io_socket/user_space/internal_listener_registry.cc @@ -18,7 +18,7 @@ void InternalListenerExtension::onServerInitialized() { server_context_.singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(internal_listener_registry), [registry = tls_registry_]() { return registry; }); - // save it in the singleton + // Save it in the singleton so the listener manager can obtain during a listener config update. ASSERT(internal_listener_registry == tls_registry_); ASSERT(internal_listener_registry->tls_slot_ == nullptr); @@ -28,9 +28,7 @@ void InternalListenerExtension::onServerInitialized() { tls_registry_->tls_slot_->set([](Event::Dispatcher&) { return std::make_shared(); }); - // This field is overwritten multiple times since onServerInitialized is called in each test case. - // ASSERT(Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ == - // nullptr); + Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ = tls_registry_->tls_slot_.get(); } diff --git a/source/extensions/io_socket/user_space/thread_local_registry.h b/source/extensions/io_socket/user_space/thread_local_registry.h index dd03ba464545a..4621d7262d354 100644 --- a/source/extensions/io_socket/user_space/thread_local_registry.h +++ b/source/extensions/io_socket/user_space/thread_local_registry.h @@ -25,7 +25,6 @@ class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, } private: - // A thread unsafe internal listener manager. Network::InternalListenerManager* manager_{nullptr}; }; } // namespace InternalListener diff --git a/source/server/BUILD b/source/server/BUILD index a9d5eec928842..19b81f8b48671 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -92,7 +92,6 @@ envoy_cc_library( "//envoy/network:listener_interface", "//envoy/runtime:runtime_interface", "//envoy/server:listener_manager_interface", - "//envoy/singleton:manager_interface", "//envoy/stats:timespan_interface", "//source/common/common:linked_object", "//source/common/common:non_copyable", diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index 40aa8ae0bd3e2..05ca70b4ae00c 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -3,9 +3,7 @@ #include #include "envoy/event/dispatcher.h" -#include "envoy/network/client_connection_factory.h" #include "envoy/network/filter.h" -#include "envoy/singleton/manager.h" #include "source/common/common/logger.h" #include "source/common/event/deferred_task.h" diff --git a/source/server/worker_impl.h b/source/server/worker_impl.h index 326f489ac2dd7..87419667eed87 100644 --- a/source/server/worker_impl.h +++ b/source/server/worker_impl.h @@ -8,7 +8,6 @@ #include "envoy/server/guarddog.h" #include "envoy/server/listener_manager.h" #include "envoy/server/worker.h" -#include "envoy/singleton/manager.h" #include "envoy/thread_local/thread_local.h" #include "source/common/common/logger.h" diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD index 009634c1605d5..f549b51982aec 100644 --- a/test/common/quic/BUILD +++ b/test/common/quic/BUILD @@ -228,7 +228,6 @@ envoy_cc_test( "//source/common/quic:envoy_quic_dispatcher_lib", "//source/common/quic:envoy_quic_proof_source_lib", "//source/common/quic:envoy_quic_server_session_lib", - "//source/common/singleton:manager_impl_lib", "//source/extensions/quic/crypto_stream:envoy_quic_crypto_server_stream_lib", "//source/server:configuration_lib", "//test/mocks/event:event_mocks", diff --git a/test/common/quic/envoy_quic_dispatcher_test.cc b/test/common/quic/envoy_quic_dispatcher_test.cc index d33629c28da6c..3481ebc731fa8 100644 --- a/test/common/quic/envoy_quic_dispatcher_test.cc +++ b/test/common/quic/envoy_quic_dispatcher_test.cc @@ -32,7 +32,6 @@ #include "test/common/quic/test_utils.h" #include "source/common/quic/envoy_quic_alarm_factory.h" #include "source/common/quic/envoy_quic_utils.h" -#include "source/common/singleton/manager_impl.h" #include "source/extensions/quic/crypto_stream/envoy_quic_crypto_server_stream.h" #include "source/server/configuration_impl.h" #include "gmock/gmock.h" diff --git a/test/extensions/common/proxy_protocol/BUILD b/test/extensions/common/proxy_protocol/BUILD index 158cad9e196eb..9acf8a7fd819a 100644 --- a/test/extensions/common/proxy_protocol/BUILD +++ b/test/extensions/common/proxy_protocol/BUILD @@ -28,7 +28,6 @@ envoy_cc_test( "//source/common/event:dispatcher_lib", "//source/common/network:connection_balancer_lib", "//source/common/network:listener_lib", - "//source/common/singleton:manager_impl_lib", "//source/extensions/common/proxy_protocol:proxy_protocol_header_lib", "//source/extensions/filters/listener/proxy_protocol:proxy_protocol_lib", "//source/server:connection_handler_lib", diff --git a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc index f00ea246a7601..0d906a0b79287 100644 --- a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc +++ b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc @@ -5,7 +5,6 @@ #include "source/common/event/dispatcher_impl.h" #include "source/common/network/connection_balancer_impl.h" #include "source/common/network/listen_socket_impl.h" -#include "source/common/singleton/manager_impl.h" #include "source/extensions/common/proxy_protocol/proxy_protocol_header.h" #include "source/extensions/filters/listener/proxy_protocol/proxy_protocol.h" #include "source/server/connection_handler_impl.h" diff --git a/test/integration/BUILD b/test/integration/BUILD index 95ac49a852257..b4c78c65ef058 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -724,7 +724,6 @@ envoy_cc_test_library( "//source/common/config:utility_lib", "//source/common/grpc:codec_lib", "//source/common/grpc:common_lib", - "//source/common/singleton:manager_impl_lib", "//source/common/http/http1:codec_lib", "//source/common/http/http2:codec_lib", "//source/common/network:connection_balancer_lib", diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 40574b085af72..acb76221012a5 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -34,7 +34,6 @@ #include "source/common/network/listen_socket_impl.h" #include "source/common/network/udp_listener_impl.h" #include "source/common/network/udp_packet_writer_handler_impl.h" -#include "source/common/singleton/manager_impl.h" #include "source/common/stats/isolated_store_impl.h" #if defined(ENVOY_ENABLE_QUIC) diff --git a/test/server/BUILD b/test/server/BUILD index e314d9436c4c1..3060862c1ef70 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -79,7 +79,6 @@ envoy_cc_test( "//source/common/network:address_lib", "//source/common/network:connection_balancer_lib", "//source/common/network:udp_packet_writer_handler_lib", - "//source/common/singleton:manager_impl_lib", "//source/common/stats:stats_lib", "//source/server:active_raw_udp_listener_config", "//source/server:connection_handler_lib", diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 11cbef74226e5..3ee3fc01a3ba4 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -7,7 +7,6 @@ #include "envoy/network/filter.h" #include "envoy/stats/scope.h" -#include "source/common/api/api_impl.h" #include "source/common/common/utility.h" #include "source/common/config/utility.h" #include "source/common/network/address_impl.h" @@ -17,7 +16,6 @@ #include "source/common/network/udp_listener_impl.h" #include "source/common/network/udp_packet_writer_handler_impl.h" #include "source/common/network/utility.h" -#include "source/common/singleton/manager_impl.h" #include "source/server/active_raw_udp_listener_config.h" #include "source/server/connection_handler_impl.h" @@ -115,7 +113,8 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable( - "internal_listener_registry_singleton", [registry = internal_registry_]() { - ENVOY_LOG_MISC(debug, "lambdai: create Dumb internal registry"); - return registry; - })); + ASSERT_NE(nullptr, server_.singletonManager().getTyped( + "internal_listener_registry_singleton", + [registry = internal_registry_]() { return registry; })); } /** * Create an IPv4 listener with a given name. diff --git a/test/server/listener_manager_impl_test.h b/test/server/listener_manager_impl_test.h index 969025bd83261..42487dc765981 100644 --- a/test/server/listener_manager_impl_test.h +++ b/test/server/listener_manager_impl_test.h @@ -309,9 +309,6 @@ class ListenerManagerImplTest : public testing::Test { class DumbInternalListenerRegistry : public Singleton::Instance, public Network::InternalListenerRegistry { public: - ~DumbInternalListenerRegistry() { - ENVOY_LOG_MISC(debug, "lambdai: destroy DumbInternalListenerRegistry"); - } MOCK_METHOD(Network::LocalInternalListenerRegistry*, getLocalRegistry, ()); }; std::shared_ptr internal_registry_{ From 7aad4378a0c5f2938e199560cffef7a9514b8b66 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 10 Jan 2022 21:09:22 +0000 Subject: [PATCH 24/59] format Signed-off-by: Yuchen Dai --- envoy/network/listener.h | 7 ++++--- .../io_socket/user_space/thread_local_registry.cc | 0 2 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 source/extensions/io_socket/user_space/thread_local_registry.cc diff --git a/envoy/network/listener.h b/envoy/network/listener.h index 0771fdff3aed3..2a4c3f97908ce 100644 --- a/envoy/network/listener.h +++ b/envoy/network/listener.h @@ -481,7 +481,7 @@ class InternalListenerManager { virtual ~InternalListenerManager() = default; /** - * Return the internal listene binding the listener address. + * Return the internal listener binding the listener address. * * @param listen_address the internal address of the expected internal listener. */ @@ -492,12 +492,13 @@ class InternalListenerManager { using InternalListenerManagerOptRef = absl::optional>; -// The thread local registry. +// The thread local registry. class LocalInternalListenerRegistry { public: virtual ~LocalInternalListenerRegistry() = default; - // Set the internal listener manager which maintains life of internal listeners. Called by connection handler. + // Set the internal listener manager which maintains life of internal listeners. Called by + // connection handler. virtual void setInternalListenerManager(InternalListenerManager& internal_listener_manager) PURE; // Get the internal listener manager to obtain a listener. Called by client connection factory. diff --git a/source/extensions/io_socket/user_space/thread_local_registry.cc b/source/extensions/io_socket/user_space/thread_local_registry.cc deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 5c430e672947ce226c50b6e912c8f22c35c587a9 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 10 Jan 2022 21:45:14 +0000 Subject: [PATCH 25/59] pedantic Signed-off-by: Yuchen Dai --- .../extensions/io_socket/user_space/client_connection_factory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/io_socket/user_space/client_connection_factory.h index c539434c97be9..e9cd696f8c814 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.h +++ b/source/extensions/io_socket/user_space/client_connection_factory.h @@ -28,7 +28,7 @@ class InternalClientConnectionFactory : public Network::ClientConnectionFactory, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) override; // The slot is owned by the internal listener registry extension. Once that extension is - // intialized, this slot is available. The ClientConnectionFactory has two potential user cases. + // initialized, this slot is available. The ClientConnectionFactory has two potential user cases. // 1. The per worker thread connection handler populates the per worker listener registry. // 2. A envoy thread local cluster lookup the per thread internal listener by listener name. // Since the population and the lookup is supposed to be executed in the same worker thread, From 2fc08540e52c0183f716d8973cfcb5cdf1cc0493 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 11 Jan 2022 00:32:57 +0000 Subject: [PATCH 26/59] intialize internal listener registry before server is intialized Signed-off-by: Yuchen Dai --- .../user_space/internal_listener_registry.cc | 16 +++++++++++++--- .../user_space/internal_listener_registry.h | 5 ++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.cc b/source/extensions/io_socket/user_space/internal_listener_registry.cc index 6dd85b46404b4..b9a847ce61933 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.cc +++ b/source/extensions/io_socket/user_space/internal_listener_registry.cc @@ -12,12 +12,22 @@ namespace InternalListener { SINGLETON_MANAGER_REGISTRATION(internal_listener_registry); +InternalListenerExtension::InternalListenerExtension( + Server::Configuration::ServerFactoryContext& server_context) + : server_context_(server_context), + tls_registry_(std::make_shared()) { + // Initialize this singleton before the listener manager potentially load a static internal + // listener. + server_context_.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(internal_listener_registry), + [registry = tls_registry_]() { return registry; }); +} + void InternalListenerExtension::onServerInitialized() { - ENVOY_LOG_MISC(debug, "lambdai: InternalListenerExtension::onServerInitialized"); std::shared_ptr internal_listener_registry = server_context_.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(internal_listener_registry), - [registry = tls_registry_]() { return registry; }); + SINGLETON_MANAGER_REGISTERED_NAME(internal_listener_registry)); + // Save it in the singleton so the listener manager can obtain during a listener config update. ASSERT(internal_listener_registry == tls_registry_); ASSERT(internal_listener_registry->tls_slot_ == nullptr); diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.h b/source/extensions/io_socket/user_space/internal_listener_registry.h index d1323944f8572..1ed44d49ec106 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.h +++ b/source/extensions/io_socket/user_space/internal_listener_registry.h @@ -28,9 +28,8 @@ class TlsInternalListenerRegistry : public Singleton::Instance, }; class InternalListenerExtension : public Server::BootstrapExtension { public: - explicit InternalListenerExtension(Server::Configuration::ServerFactoryContext& server_context) - : server_context_(server_context), - tls_registry_(std::make_shared()) {} + explicit InternalListenerExtension(Server::Configuration::ServerFactoryContext& server_context); + ~InternalListenerExtension() override = default; // Server::Configuration::BootstrapExtension From abc3efe117f0eaf2ad1c05fad256e122df831d2f Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 11 Jan 2022 00:34:28 +0000 Subject: [PATCH 27/59] fill an envoy internal source address Signed-off-by: Yuchen Dai --- .../io_socket/user_space/client_connection_factory.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index 11a28cf0d2fa4..4c835ba9b0110 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -21,6 +21,12 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec Network::Address::InstanceConstSharedPtr source_address, Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) { + // OS does not fill the address automatically so a pivotal address is populated. + // TODO(lambdai): provide option to fill the downstream remote address here. + if (source_address == nullptr) { + source_address = + std::make_shared("internal_client_address"); + } auto [io_handle_client, io_handle_server] = Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); From 13c7a52fd8e1b0f0e2629d4938a63f8f09a48a0a Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 11 Jan 2022 16:51:15 +0000 Subject: [PATCH 28/59] clang_tidy Signed-off-by: Yuchen Dai --- test/server/connection_handler_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 3ee3fc01a3ba4..f1a354c156fdc 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -137,7 +137,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable Date: Tue, 11 Jan 2022 20:36:19 +0000 Subject: [PATCH 29/59] set default 1MB buffer size for user space handle in connection factory Signed-off-by: Yuchen Dai --- .../io_socket/user_space/client_connection_factory.cc | 5 ++++- .../extensions/io_socket/user_space/io_handle_impl.h | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index 4c835ba9b0110..44f154a8ef677 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -29,7 +29,10 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec } auto [io_handle_client, io_handle_server] = - Extensions::IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); + // 1MB is the default connection buffer size. + // TODO(lambdai): export as configuration. + Extensions::IoSocket::UserSpace::IoHandleFactory::createBufferLimitedIoHandlePair(1024 * + 1024); auto client_conn = std::make_unique( dispatcher, diff --git a/source/extensions/io_socket/user_space/io_handle_impl.h b/source/extensions/io_socket/user_space/io_handle_impl.h index 9f7ab9010b8b4..b5aa6b7505ced 100644 --- a/source/extensions/io_socket/user_space/io_handle_impl.h +++ b/source/extensions/io_socket/user_space/io_handle_impl.h @@ -185,6 +185,17 @@ class IoHandleFactory { p.second->setPeerHandle(p.first.get()); return p; } + static std::pair + createBufferLimitedIoHandlePair(uint32_t buffer_size) { + auto p = std::pair{new IoHandleImpl(), new IoHandleImpl()}; + // This buffer watermark setting emulates the OS socket buffer parameter + // `/proc/sys/net/ipv4/tcp_{r,w}mem`. + p.first->setWatermarks(buffer_size); + p.second->setWatermarks(buffer_size); + p.first->setPeerHandle(p.second.get()); + p.second->setPeerHandle(p.first.get()); + return p; + } }; } // namespace UserSpace } // namespace IoSocket From ee2d67f580d72af5149a6a0724d54b6b6cb19376 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 24 Jan 2022 23:35:58 +0000 Subject: [PATCH 30/59] category and add TODO Signed-off-by: Yuchen Dai --- envoy/network/client_connection_factory.h | 2 +- source/common/event/dispatcher_impl.cc | 5 +++-- .../common/network/default_client_connection_factory.h | 10 +++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/envoy/network/client_connection_factory.h b/envoy/network/client_connection_factory.h index defffd1f7685a..c9a69b2ca4f43 100644 --- a/envoy/network/client_connection_factory.h +++ b/envoy/network/client_connection_factory.h @@ -16,7 +16,7 @@ class ClientConnectionFactory : public Config::UntypedFactory { ~ClientConnectionFactory() override = default; // Config::UntypedFactory - std::string category() const override { return "network.connection"; } + std::string category() const override { return "network.connection.client"; } /** * @param address The target remote address. diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 72d8029c27af3..c94e5f44f85ac 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -154,8 +154,9 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr auto* factory = Config::Utility::getFactoryByName( std::string(Network::Address::addressType(address))); - // The target address is usually offered by EDS and the EDS api should reject the supported - // address. Alternatively, we can create a closed connection to avoid crash. Note that the caller + // The target address is usually offered by EDS and the EDS api should reject the unsupported + // address. + // TODO(lambdai): Return a closed connection if the factory is not found. Note that the caller // expects a non-null connection as of today so we cannot gracefully handle unsupported address // type. return factory->createClientConnection(*this, address, source_address, diff --git a/source/common/network/default_client_connection_factory.h b/source/common/network/default_client_connection_factory.h index a4f96e29e267b..6658e2d3093c0 100644 --- a/source/common/network/default_client_connection_factory.h +++ b/source/common/network/default_client_connection_factory.h @@ -8,10 +8,18 @@ namespace Envoy { namespace Network { -class DefaultClientConnectionFactory : public Network::ClientConnectionFactory { +/** + * This client connection factory handles the connection if the remote address type is either ip or + * pipe. + */ +class DefaultClientConnectionFactory : public ClientConnectionFactory { public: ~DefaultClientConnectionFactory() override = default; + + // Config::UntypedFactory std::string name() const override { return "default"; } + + // Network::ClientConnectionFactory Network::ClientConnectionPtr createClientConnection(Event::Dispatcher& dispatcher, Network::Address::InstanceConstSharedPtr address, From e9bd554245a4d20df0fe298f654a60d2ca5da296 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 25 Jan 2022 00:32:22 +0000 Subject: [PATCH 31/59] move addressType into address instance Signed-off-by: Yuchen Dai --- envoy/network/address.h | 6 ++++++ source/common/event/dispatcher_impl.cc | 2 +- source/common/network/address_impl.cc | 12 ------------ source/common/network/address_impl.h | 10 ++++------ .../network/dns_resolver/cares/dns_impl_test.cc | 2 ++ test/mocks/network/mocks.h | 1 + 6 files changed, 14 insertions(+), 19 deletions(-) diff --git a/envoy/network/address.h b/envoy/network/address.h index 2b50b0728168e..6339535942109 100644 --- a/envoy/network/address.h +++ b/envoy/network/address.h @@ -201,6 +201,12 @@ class Instance { */ virtual Type type() const PURE; + /** + * Return the address type in string_view. The returned type name is used to find the + * ClientConnectionFactory. + */ + virtual absl::string_view addressType() const PURE; + /** * @return SocketInterface to be used with the address. */ diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index c94e5f44f85ac..bb45b279a8101 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -153,7 +153,7 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr ASSERT(isThreadSafe()); auto* factory = Config::Utility::getFactoryByName( - std::string(Network::Address::addressType(address))); + std::string(address->addressType())); // The target address is usually offered by EDS and the EDS api should reject the unsupported // address. // TODO(lambdai): Return a closed connection if the factory is not found. Note that the caller diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index a29566506109d..84775e01bfdb2 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -117,18 +117,6 @@ addressFromSockAddrOrDie(const sockaddr_storage& ss, socklen_t ss_len, os_fd_t f return *address; } -absl::string_view addressType(const Network::Address::InstanceConstSharedPtr& addr) { - ASSERT(addr != nullptr); - switch (addr->type()) { - case Network::Address::Type::Ip: - case Network::Address::Type::Pipe: - return "default"; - case Network::Address::Type::EnvoyInternal: - return "EnvoyInternal"; - } - NOT_REACHED_GCOVR_EXCL_LINE; -}; - Ipv4Instance::Ipv4Instance(const sockaddr_in* address, const SocketInterface* sock_interface) : InstanceBase(Type::Ip, sockInterfaceOrDefault(sock_interface)) { throwOnError(validateProtocolSupported()); diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index 76bf59ccddadc..918bdc7b2a4b7 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -44,12 +44,6 @@ InstanceConstSharedPtr addressFromSockAddrOrThrow(const sockaddr_storage& ss, so InstanceConstSharedPtr addressFromSockAddrOrDie(const sockaddr_storage& ss, socklen_t ss_len, os_fd_t fd, bool v6only = true); -/** - * Return the address type in string_view. The returned type name is used by calling - * ClientConnectionFactory. - */ -absl::string_view addressType(const Network::Address::InstanceConstSharedPtr& addr); - /** * Base class for all address types. */ @@ -129,6 +123,7 @@ class Ipv4Instance : public InstanceBase { return reinterpret_cast(&ip_.ipv4_.address_); } socklen_t sockAddrLen() const override { return sizeof(sockaddr_in); } + absl::string_view addressType() const override { return "default"; } /** * Convenience function to convert an IPv4 address to canonical string format. @@ -219,6 +214,7 @@ class Ipv6Instance : public InstanceBase { return reinterpret_cast(&ip_.ipv6_.address_); } socklen_t sockAddrLen() const override { return sizeof(sockaddr_in6); } + absl::string_view addressType() const override { return "default"; } // Validate that IPv6 is supported on this platform static absl::Status validateProtocolSupported(); @@ -305,6 +301,7 @@ class PipeInstance : public InstanceBase { } return sizeof(pipe_.address_); } + absl::string_view addressType() const override { return "default"; } private: /** @@ -350,6 +347,7 @@ class EnvoyInternalInstance : public InstanceBase { // TODO(lambdai): Verify all callers accepts nullptr. const sockaddr* sockAddr() const override { return nullptr; } socklen_t sockAddrLen() const override { return 0; } + absl::string_view addressType() const override { return "EnvoyInternal"; } private: struct EnvoyInternalAddressImpl : public EnvoyInternalAddress { diff --git a/test/extensions/network/dns_resolver/cares/dns_impl_test.cc b/test/extensions/network/dns_resolver/cares/dns_impl_test.cc index b22dd0383623d..70914d4f50510 100644 --- a/test/extensions/network/dns_resolver/cares/dns_impl_test.cc +++ b/test/extensions/network/dns_resolver/cares/dns_impl_test.cc @@ -573,6 +573,8 @@ class CustomInstance : public Address::Instance { } const sockaddr* sockAddr() const override { return instance_.sockAddr(); } socklen_t sockAddrLen() const override { return instance_.sockAddrLen(); } + absl::string_view addressType() const override { PANIC("not implemented"); } + Address::Type type() const override { return instance_.type(); } const SocketInterface& socketInterface() const override { return SocketInterfaceSingleton::get(); diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 40e5c43edd2ad..c84a02f4e3c2a 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -520,6 +520,7 @@ class MockResolvedAddress : public Address::Instance { MOCK_METHOD(Address::Type, type, (), (const)); MOCK_METHOD(const sockaddr*, sockAddr, (), (const)); MOCK_METHOD(socklen_t, sockAddrLen, (), (const)); + MOCK_METHOD(absl::string_view, addressType, (), (const)); const std::string& asString() const override { return physical_; } absl::string_view asStringView() const override { return physical_; } From cc8efbc5a901c9a75dfb485b5cbb18a1ee440060 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 3 Feb 2022 22:36:13 +0000 Subject: [PATCH 32/59] define api for internal connection registry Signed-off-by: Yuchen Dai --- .../io_socket/user_space/v3/bootstrap.proto | 20 +++++++++++++++++++ api/versioning/BUILD | 1 + source/extensions/io_socket/user_space/BUILD | 1 + .../user_space/internal_listener_registry.cc | 3 ++- .../user_space/internal_listener_registry.h | 4 +++- .../internal_listener_integration_test.cc | 2 +- 6 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto diff --git a/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto b/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto new file mode 100644 index 0000000000000..9ad5b1526ba48 --- /dev/null +++ b/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package envoy.extensions.io_socket.user_space.v3; + +import "google/protobuf/any.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.io_socket.user_space.v3"; +option java_outer_classname = "BootstrapProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/io_socket/user_space/v3;user_spacev3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: InternalConnectionRegistry] +// [#extension: envoy.io_socket.user_space] + +message InternalConnectionRegistry { +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index acfd91b63b4dd..3b1c112842c56 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -152,6 +152,7 @@ proto_library( "//envoy/extensions/internal_redirect/allow_listed_routes/v3:pkg", "//envoy/extensions/internal_redirect/previous_routes/v3:pkg", "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", + "//envoy/extensions/io_socket/user_space/v3:pkg", "//envoy/extensions/key_value/file_based/v3:pkg", "//envoy/extensions/matching/common_inputs/environment_variable/v3:pkg", "//envoy/extensions/matching/input_matchers/consistent_hashing/v3:pkg", diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 9b9f1153b0dcf..346bf67659f57 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -92,6 +92,7 @@ envoy_cc_library( deps = [ ":thread_local_registry", "//envoy/server:bootstrap_extension_config_interface", + "@envoy_api//envoy/extensions/io_socket/user_space/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.cc b/source/extensions/io_socket/user_space/internal_listener_registry.cc index b9a847ce61933..91730478f08ef 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.cc +++ b/source/extensions/io_socket/user_space/internal_listener_registry.cc @@ -44,7 +44,8 @@ void InternalListenerExtension::onServerInitialized() { } Server::BootstrapExtensionPtr InternalListenerRegistryFactory::createBootstrapExtension( - const Protobuf::Message&, Server::Configuration::ServerFactoryContext& context) { + [[maybe_unused]] const Protobuf::Message& config, + Server::Configuration::ServerFactoryContext& context) { return std::make_unique(context); } diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.h b/source/extensions/io_socket/user_space/internal_listener_registry.h index 1ed44d49ec106..d1fcfb68bb38b 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.h +++ b/source/extensions/io_socket/user_space/internal_listener_registry.h @@ -1,6 +1,7 @@ #pragma once #include "envoy/config/typed_config.h" +#include "envoy/extensions/io_socket/user_space/v3/bootstrap.pb.h" #include "envoy/registry/registry.h" #include "envoy/server/bootstrap_extension_config.h" @@ -46,7 +47,8 @@ class InternalListenerRegistryFactory : public Server::Configuration::BootstrapE createBootstrapExtension(const Protobuf::Message& config, Server::Configuration::ServerFactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override { - return std::make_unique(); + return std::make_unique< + envoy::extensions::io_socket::user_space::v3::InternalConnectionRegistry>(); } std::string name() const override { return "envoy.bootstrap.internal_listener_registry"; }; }; diff --git a/test/integration/internal_listener_integration_test.cc b/test/integration/internal_listener_integration_test.cc index 38bad718564b2..fe73d98283191 100644 --- a/test/integration/internal_listener_integration_test.cc +++ b/test/integration/internal_listener_integration_test.cc @@ -35,7 +35,7 @@ class InternalListenerIntegrationTest : public testing::TestWithParam Date: Fri, 4 Feb 2022 18:08:57 +0000 Subject: [PATCH 33/59] add missing BUILD file in git Signed-off-by: Yuchen Dai --- api/envoy/extensions/io_socket/user_space/v3/BUILD | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 api/envoy/extensions/io_socket/user_space/v3/BUILD diff --git a/api/envoy/extensions/io_socket/user_space/v3/BUILD b/api/envoy/extensions/io_socket/user_space/v3/BUILD new file mode 100644 index 0000000000000..ee92fb652582e --- /dev/null +++ b/api/envoy/extensions/io_socket/user_space/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) From e4894cdf5036ef7e0c50163a7bbbef181b483475 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 8 Feb 2022 21:21:47 +0000 Subject: [PATCH 34/59] amend api Signed-off-by: Yuchen Dai --- .../io_socket/user_space/v3/bootstrap.proto | 10 ++++++++-- source/common/network/address_impl.h | 2 +- .../user_space/client_connection_factory.cc | 2 +- .../user_space/client_connection_factory.h | 5 ++--- .../user_space/internal_listener_registry.cc | 12 ++++++------ .../user_space/internal_listener_registry.h | 14 ++++++-------- .../io_socket/user_space/thread_local_registry.h | 6 ++++-- .../user_space/client_connection_factory_test.cc | 12 +++++------- .../internal_listener_integration_test.cc | 2 +- 9 files changed, 34 insertions(+), 31 deletions(-) diff --git a/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto b/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto index 9ad5b1526ba48..5266506c9e74f 100644 --- a/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto +++ b/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto @@ -13,8 +13,14 @@ option java_multiple_files = true; option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/io_socket/user_space/v3;user_spacev3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; -// [#protodoc-title: InternalConnectionRegistry] +// [#protodoc-title: Internal Listener] +// `InternalListenerRegistry` message is the configuration of the `envoy.bootstrap.internal_listener_registry` bootstrap extension. +// This boot strap extension is required to support adding a listener with an internal address. +// To establish a connection to a registered internal listener, another `envoy.io_socket.user_space` extension is required. +// The `InternalClientConnectionFactory` within enables the dispatcher to create client connection to any envoy internal address. // [#extension: envoy.io_socket.user_space] -message InternalConnectionRegistry { +// [#not-implemented-hide:] +// Configuration for internal listener registry. +message InternalListenerRegistry { } diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index 918bdc7b2a4b7..20a932b259fcb 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -347,7 +347,7 @@ class EnvoyInternalInstance : public InstanceBase { // TODO(lambdai): Verify all callers accepts nullptr. const sockaddr* sockAddr() const override { return nullptr; } socklen_t sockAddrLen() const override { return 0; } - absl::string_view addressType() const override { return "EnvoyInternal"; } + absl::string_view addressType() const override { return "envoy_internal"; } private: struct EnvoyInternalAddressImpl : public EnvoyInternalAddress { diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/io_socket/user_space/client_connection_factory.cc index 44f154a8ef677..8168fc00ce704 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/io_socket/user_space/client_connection_factory.cc @@ -13,7 +13,7 @@ namespace Extensions { namespace IoSocket { namespace UserSpace { -ThreadLocal::TypedSlot* +ThreadLocal::TypedSlot* InternalClientConnectionFactory::registry_tls_slot_ = nullptr; Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnection( diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/io_socket/user_space/client_connection_factory.h index e9cd696f8c814..44392ae77d08d 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.h +++ b/source/extensions/io_socket/user_space/client_connection_factory.h @@ -20,7 +20,7 @@ class InternalClientConnectionFactory : public Network::ClientConnectionFactory, Logger::Loggable { public: ~InternalClientConnectionFactory() override = default; - std::string name() const override { return "EnvoyInternal"; } + std::string name() const override { return "envoy_internal"; } Network::ClientConnectionPtr createClientConnection(Event::Dispatcher& dispatcher, Network::Address::InstanceConstSharedPtr address, @@ -34,8 +34,7 @@ class InternalClientConnectionFactory : public Network::ClientConnectionFactory, // Since the population and the lookup is supposed to be executed in the same worker thread, // neither need to hold a lock. // TODO(lambdai): make it friend to only bootstrap extension. - static ThreadLocal::TypedSlot* - registry_tls_slot_; + static ThreadLocal::TypedSlot* registry_tls_slot_; }; } // namespace UserSpace diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.cc b/source/extensions/io_socket/user_space/internal_listener_registry.cc index 91730478f08ef..6bade3cb203d9 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.cc +++ b/source/extensions/io_socket/user_space/internal_listener_registry.cc @@ -7,8 +7,8 @@ namespace Envoy { namespace Extensions { -namespace Bootstrap { -namespace InternalListener { +namespace IoSocket { +namespace UserSpace { SINGLETON_MANAGER_REGISTRATION(internal_listener_registry); @@ -33,10 +33,10 @@ void InternalListenerExtension::onServerInitialized() { ASSERT(internal_listener_registry->tls_slot_ == nullptr); tls_registry_->tls_slot_ = - ThreadLocal::TypedSlot::makeUnique( + ThreadLocal::TypedSlot::makeUnique( server_context_.threadLocal()); tls_registry_->tls_slot_->set([](Event::Dispatcher&) { - return std::make_shared(); + return std::make_shared(); }); Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ = @@ -51,7 +51,7 @@ Server::BootstrapExtensionPtr InternalListenerRegistryFactory::createBootstrapEx REGISTER_FACTORY(InternalListenerRegistryFactory, Server::Configuration::BootstrapExtensionFactory); -} // namespace InternalListener -} // namespace Bootstrap +} // namespace UserSpace +} // namespace IoSocket } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.h b/source/extensions/io_socket/user_space/internal_listener_registry.h index d1fcfb68bb38b..291131c2fd908 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.h +++ b/source/extensions/io_socket/user_space/internal_listener_registry.h @@ -11,9 +11,8 @@ namespace Envoy { namespace Extensions { -namespace Bootstrap { -namespace InternalListener { - +namespace IoSocket { +namespace UserSpace { class TlsInternalListenerRegistry : public Singleton::Instance, public Network::InternalListenerRegistry { public: @@ -24,8 +23,7 @@ class TlsInternalListenerRegistry : public Singleton::Instance, return nullptr; } - std::unique_ptr> - tls_slot_; + std::unique_ptr> tls_slot_; }; class InternalListenerExtension : public Server::BootstrapExtension { public: @@ -48,12 +46,12 @@ class InternalListenerRegistryFactory : public Server::Configuration::BootstrapE Server::Configuration::ServerFactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique< - envoy::extensions::io_socket::user_space::v3::InternalConnectionRegistry>(); + envoy::extensions::io_socket::user_space::v3::InternalListenerRegistry>(); } std::string name() const override { return "envoy.bootstrap.internal_listener_registry"; }; }; -} // namespace InternalListener -} // namespace Bootstrap +} // namespace UserSpace +} // namespace IoSocket } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/io_socket/user_space/thread_local_registry.h b/source/extensions/io_socket/user_space/thread_local_registry.h index 4621d7262d354..6770e4d285b94 100644 --- a/source/extensions/io_socket/user_space/thread_local_registry.h +++ b/source/extensions/io_socket/user_space/thread_local_registry.h @@ -5,7 +5,8 @@ namespace Envoy { namespace Extensions { -namespace InternalListener { +namespace IoSocket { +namespace UserSpace { class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, public Network::LocalInternalListenerRegistry { @@ -27,6 +28,7 @@ class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, private: Network::InternalListenerManager* manager_{nullptr}; }; -} // namespace InternalListener +} // namespace UserSpace +} // namespace IoSocket } // namespace Extensions } // namespace Envoy diff --git a/test/extensions/io_socket/user_space/client_connection_factory_test.cc b/test/extensions/io_socket/user_space/client_connection_factory_test.cc index d5c41b021bba3..ac97b7e02619d 100644 --- a/test/extensions/io_socket/user_space/client_connection_factory_test.cc +++ b/test/extensions/io_socket/user_space/client_connection_factory_test.cc @@ -40,9 +40,8 @@ class ClientConnectionFactoryTest : public testing::Test { buf_(1024) { std::tie(io_handle_, io_handle_peer_) = IoHandleFactory::createIoHandlePair(); EXPECT_CALL(tls_allocator_, allocateSlot()); - tls_slot_ = - ThreadLocal::TypedSlot::makeUnique( - tls_allocator_); + tls_slot_ = ThreadLocal::TypedSlot::makeUnique( + tls_allocator_); tls_slot_->set([r = registry_](Event::Dispatcher&) { return r; }); // TODO: restore the original value via RAII. Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ = @@ -52,11 +51,10 @@ class ClientConnectionFactoryTest : public testing::Test { Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; MockInternalListenerManger internal_listener_manager_; - std::shared_ptr registry_{ - std::make_shared()}; + std::shared_ptr registry_{ + std::make_shared()}; ThreadLocal::MockInstance tls_allocator_; - std::unique_ptr> - tls_slot_; + std::unique_ptr> tls_slot_; // Owned by IoHandleImpl. NiceMock* schedulable_cb_; diff --git a/test/integration/internal_listener_integration_test.cc b/test/integration/internal_listener_integration_test.cc index fe73d98283191..9f223d35a751d 100644 --- a/test/integration/internal_listener_integration_test.cc +++ b/test/integration/internal_listener_integration_test.cc @@ -35,7 +35,7 @@ class InternalListenerIntegrationTest : public testing::TestWithParam Date: Wed, 9 Feb 2022 23:47:29 +0000 Subject: [PATCH 35/59] proto comment, cont Signed-off-by: Yuchen Dai --- api/envoy/config/core/v3/address.proto | 2 +- .../io_socket/user_space/v3/bootstrap.proto | 20 +++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/api/envoy/config/core/v3/address.proto b/api/envoy/config/core/v3/address.proto index df5001bdd4f24..8813ab4dffc51 100644 --- a/api/envoy/config/core/v3/address.proto +++ b/api/envoy/config/core/v3/address.proto @@ -31,7 +31,7 @@ message Pipe { uint32 mode = 2 [(validate.rules).uint32 = {lte: 511}]; } -// [#not-implemented-hide:] The address represents an envoy internal listener. +// The address represents an envoy internal listener. // TODO(lambdai): Make this address available for listener and endpoint. // TODO(asraa): When address available, remove workaround from test/server/server_fuzz_test.cc:30. message EnvoyInternalAddress { diff --git a/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto b/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto index 5266506c9e74f..c3c653a89080f 100644 --- a/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto +++ b/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto @@ -14,13 +14,25 @@ option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/io_ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Internal Listener] -// `InternalListenerRegistry` message is the configuration of the `envoy.bootstrap.internal_listener_registry` bootstrap extension. -// This boot strap extension is required to support adding a listener with an internal address. -// To establish a connection to a registered internal listener, another `envoy.io_socket.user_space` extension is required. -// The `InternalClientConnectionFactory` within enables the dispatcher to create client connection to any envoy internal address. +// +// This extension contains 2 major components to add a listener with +// an :ref:`envoy internal address ` +// and to create a client connection to that :ref:`listener ` +// +// 1. The bootstrap extension `envoy.bootstrap.internal_listener_registry` +// +// This bootstrap extension is required to support looking up the target listener via an +// :ref:`envoy internal address ` on each worker threads. +// +// 2. The client connection factory `network.connection.client.envoy_internal` +// +// This factory is implicitly instantiated by the dispatcher to establish a client connection to the above +// internal listener, +// // [#extension: envoy.io_socket.user_space] // [#not-implemented-hide:] // Configuration for internal listener registry. +// [#extension-category: envoy.bootstrap] message InternalListenerRegistry { } From 35fb6fd378d336a908449388966fede7867afce0 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 10 Feb 2022 18:18:01 +0000 Subject: [PATCH 36/59] more comments Signed-off-by: Yuchen Dai --- .../io_socket/user_space/client_connection_factory.h | 1 + .../io_socket/user_space/internal_listener_registry.cc | 4 ++++ .../io_socket/user_space/internal_listener_registry.h | 7 +++++++ .../io_socket/user_space/thread_local_registry.h | 5 +++++ 4 files changed, 17 insertions(+) diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/io_socket/user_space/client_connection_factory.h index 44392ae77d08d..f62e383066bae 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.h +++ b/source/extensions/io_socket/user_space/client_connection_factory.h @@ -16,6 +16,7 @@ namespace Extensions { namespace IoSocket { namespace UserSpace { +// This factory creates the client connection to an envoy internal address. class InternalClientConnectionFactory : public Network::ClientConnectionFactory, Logger::Loggable { public: diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.cc b/source/extensions/io_socket/user_space/internal_listener_registry.cc index 6bade3cb203d9..42603abc27d63 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.cc +++ b/source/extensions/io_socket/user_space/internal_listener_registry.cc @@ -39,6 +39,10 @@ void InternalListenerExtension::onServerInitialized() { return std::make_shared(); }); + // Now the thread local registry is available. This thread local object is published to + // ``InternalClientConnectionFactory``. + // Note that the per silo ``ConnectionHandler`` will add internal listeners into the per silo + // registry. Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ = tls_registry_->tls_slot_.get(); } diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.h b/source/extensions/io_socket/user_space/internal_listener_registry.h index 291131c2fd908..b197307a41099 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.h +++ b/source/extensions/io_socket/user_space/internal_listener_registry.h @@ -13,6 +13,8 @@ namespace Envoy { namespace Extensions { namespace IoSocket { namespace UserSpace { + +// This InternalListenerRegistry implementation owns a thread local slot. class TlsInternalListenerRegistry : public Singleton::Instance, public Network::InternalListenerRegistry { public: @@ -25,6 +27,10 @@ class TlsInternalListenerRegistry : public Singleton::Instance, std::unique_ptr> tls_slot_; }; + +// This extension maintains maintain the life of the ``TlsInternalListenerRegistry`` singleton. +// The ``TlsInternalListenerRegistry`` is functionally ready after the Envoy thread local system is +// initialized. class InternalListenerExtension : public Server::BootstrapExtension { public: explicit InternalListenerExtension(Server::Configuration::ServerFactoryContext& server_context); @@ -38,6 +44,7 @@ class InternalListenerExtension : public Server::BootstrapExtension { std::shared_ptr tls_registry_; }; +// The factory creates the `InternalListenerExtension` instance when envoy starts. class InternalListenerRegistryFactory : public Server::Configuration::BootstrapExtensionFactory { public: // Server::Configuration::BootstrapExtensionFactory diff --git a/source/extensions/io_socket/user_space/thread_local_registry.h b/source/extensions/io_socket/user_space/thread_local_registry.h index 6770e4d285b94..5859b551638d0 100644 --- a/source/extensions/io_socket/user_space/thread_local_registry.h +++ b/source/extensions/io_socket/user_space/thread_local_registry.h @@ -8,11 +8,13 @@ namespace Extensions { namespace IoSocket { namespace UserSpace { +// The registry is constructed and accessed on each silo. class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, public Network::LocalInternalListenerRegistry { public: ThreadLocalRegistryImpl() = default; + // Network::LocalInternalListenerRegistry void setInternalListenerManager(Network::InternalListenerManager& internal_listener_manager) override { manager_ = &internal_listener_manager; @@ -20,12 +22,15 @@ class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, Network::InternalListenerManagerOptRef getInternalListenerManager() override { if (manager_ == nullptr) { + // The internal listener manager is published to this registry when the first internal + // listener is added through LDS. Return null prior to this moment. return Network::InternalListenerManagerOptRef(); } return Network::InternalListenerManagerOptRef(*manager_); } private: + // The typical instance is the ``ConnectionHandlerImpl`` on the same thread. Network::InternalListenerManager* manager_{nullptr}; }; } // namespace UserSpace From c41de728c5d32b844cada178c7d894fee77f62f5 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 11 Feb 2022 19:30:03 +0000 Subject: [PATCH 37/59] rename Signed-off-by: Yuchen Dai --- .../user_space/v3/{bootstrap.proto => internal_listener.proto} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename api/envoy/extensions/io_socket/user_space/v3/{bootstrap.proto => internal_listener.proto} (96%) diff --git a/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto b/api/envoy/extensions/io_socket/user_space/v3/internal_listener.proto similarity index 96% rename from api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto rename to api/envoy/extensions/io_socket/user_space/v3/internal_listener.proto index c3c653a89080f..d03bee361f7d7 100644 --- a/api/envoy/extensions/io_socket/user_space/v3/bootstrap.proto +++ b/api/envoy/extensions/io_socket/user_space/v3/internal_listener.proto @@ -8,7 +8,7 @@ import "udpa/annotations/status.proto"; import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.extensions.io_socket.user_space.v3"; -option java_outer_classname = "BootstrapProto"; +option java_outer_classname = "InternalListenerProto"; option java_multiple_files = true; option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/io_socket/user_space/v3;user_spacev3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; From e9c348c2da1526de882c36343b51ffdcc74aef0d Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 11 Feb 2022 19:41:01 +0000 Subject: [PATCH 38/59] rename, cont Signed-off-by: Yuchen Dai --- .../io_socket/user_space/internal_listener_registry.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.h b/source/extensions/io_socket/user_space/internal_listener_registry.h index b197307a41099..ec0e59b04a1b9 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.h +++ b/source/extensions/io_socket/user_space/internal_listener_registry.h @@ -1,7 +1,7 @@ #pragma once #include "envoy/config/typed_config.h" -#include "envoy/extensions/io_socket/user_space/v3/bootstrap.pb.h" +#include "envoy/extensions/io_socket/user_space/v3/internal_listener.pb.h" #include "envoy/registry/registry.h" #include "envoy/server/bootstrap_extension_config.h" From 33b11e54e4f07b32d5c89cbbe012d31186112694 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 00:41:41 +0000 Subject: [PATCH 39/59] add boostrap.internal_listener_registry extension Signed-off-by: Yuchen Dai --- CODEOWNERS | 3 +- .../internal_listener_registry}/v3/BUILD | 0 .../v3/internal_listener.proto | 15 ++--- api/versioning/BUILD | 2 +- .../internal_listener_registry/BUILD | 67 +++++++++++++++++++ .../client_connection_factory.cc | 12 ++-- .../client_connection_factory.h | 14 ++-- .../internal_listener_registry.cc | 22 +++--- .../internal_listener_registry.h | 18 ++--- .../thread_local_registry.h | 8 +-- source/extensions/extensions_build_config.bzl | 2 +- source/extensions/extensions_metadata.yaml | 5 ++ source/extensions/io_socket/user_space/BUILD | 51 +------------- .../internal_listener_registry/BUILD | 26 +++++++ .../client_connection_factory_test.cc | 28 ++++---- test/extensions/io_socket/user_space/BUILD | 14 ---- test/integration/BUILD | 4 +- 17 files changed, 164 insertions(+), 127 deletions(-) rename api/envoy/extensions/{io_socket/user_space => bootstrap/internal_listener_registry}/v3/BUILD (100%) rename api/envoy/extensions/{io_socket/user_space => bootstrap/internal_listener_registry}/v3/internal_listener.proto (72%) create mode 100644 source/extensions/bootstrap/internal_listener_registry/BUILD rename source/extensions/{io_socket/user_space => bootstrap/internal_listener_registry}/client_connection_factory.cc (90%) rename source/extensions/{io_socket/user_space => bootstrap/internal_listener_registry}/client_connection_factory.h (83%) rename source/extensions/{io_socket/user_space => bootstrap/internal_listener_registry}/internal_listener_registry.cc (74%) rename source/extensions/{io_socket/user_space => bootstrap/internal_listener_registry}/internal_listener_registry.h (78%) rename source/extensions/{io_socket/user_space => bootstrap/internal_listener_registry}/thread_local_registry.h (90%) create mode 100644 test/extensions/bootstrap/internal_listener_registry/BUILD rename test/extensions/{io_socket/user_space => bootstrap/internal_listener_registry}/client_connection_factory_test.cc (85%) diff --git a/CODEOWNERS b/CODEOWNERS index cf21279c7e0a7..e4a53110c510f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -170,8 +170,9 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp /*/extensions/matching/input_matchers/consistent_hashing @snowp @donyu # environment generic input /*/extensions/matching/common_inputs/environment @snowp @donyu -# user space socket pair and event +# user space socket pair, event, connection and listener /*/extensions/io_socket/user_space @lambdai @antoniovicente +/*/extensions/bootstrap/internal_listener_registry @lambdai @adisuissa # Default UUID4 request ID extension /*/extensions/request_id/uuid @mattklein123 @alyssawilk # HTTP header formatters diff --git a/api/envoy/extensions/io_socket/user_space/v3/BUILD b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/BUILD similarity index 100% rename from api/envoy/extensions/io_socket/user_space/v3/BUILD rename to api/envoy/extensions/bootstrap/internal_listener_registry/v3/BUILD diff --git a/api/envoy/extensions/io_socket/user_space/v3/internal_listener.proto b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto similarity index 72% rename from api/envoy/extensions/io_socket/user_space/v3/internal_listener.proto rename to api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto index d03bee361f7d7..ea95087b2c039 100644 --- a/api/envoy/extensions/io_socket/user_space/v3/internal_listener.proto +++ b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto @@ -1,19 +1,18 @@ syntax = "proto3"; -package envoy.extensions.io_socket.user_space.v3; - -import "google/protobuf/any.proto"; +package envoy.extensions.bootstrap.internal_listener_registry.v3; import "udpa/annotations/status.proto"; -import "validate/validate.proto"; +import "xds/annotations/v3/status.proto"; -option java_package = "io.envoyproxy.envoy.extensions.io_socket.user_space.v3"; +option java_package = "io.envoyproxy.envoy.extensions.bootstrap.internal_listener_registry.v3"; option java_outer_classname = "InternalListenerProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/io_socket/user_space/v3;user_spacev3"; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/bootstrap/internal_listener_registry/v3;internal_listener_registryv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; +option (xds.annotations.v3.file_status).work_in_progress = true; -// [#protodoc-title: Internal Listener] +// [#protodoc-title: InternalListenerRegistry] // // This extension contains 2 major components to add a listener with // an :ref:`envoy internal address ` @@ -29,7 +28,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // This factory is implicitly instantiated by the dispatcher to establish a client connection to the above // internal listener, // -// [#extension: envoy.io_socket.user_space] +// [#extension: envoy.bootstrap.internal_listener_registry] // [#not-implemented-hide:] // Configuration for internal listener registry. diff --git a/api/versioning/BUILD b/api/versioning/BUILD index bfcdb8ec72c7e..7611530daf128 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -56,6 +56,7 @@ proto_library( "//envoy/extensions/access_loggers/open_telemetry/v3:pkg", "//envoy/extensions/access_loggers/stream/v3:pkg", "//envoy/extensions/access_loggers/wasm/v3:pkg", + "//envoy/extensions/bootstrap/internal_listener_registry/v3:pkg", "//envoy/extensions/cache/simple_http_cache/v3:pkg", "//envoy/extensions/clusters/aggregate/v3:pkg", "//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg", @@ -155,7 +156,6 @@ proto_library( "//envoy/extensions/internal_redirect/allow_listed_routes/v3:pkg", "//envoy/extensions/internal_redirect/previous_routes/v3:pkg", "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", - "//envoy/extensions/io_socket/user_space/v3:pkg", "//envoy/extensions/key_value/file_based/v3:pkg", "//envoy/extensions/load_balancing_policies/round_robin/v3:pkg", "//envoy/extensions/load_balancing_policies/wrr_locality/v3:pkg", diff --git a/source/extensions/bootstrap/internal_listener_registry/BUILD b/source/extensions/bootstrap/internal_listener_registry/BUILD new file mode 100644 index 0000000000000..820cf4c6a256b --- /dev/null +++ b/source/extensions/bootstrap/internal_listener_registry/BUILD @@ -0,0 +1,67 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "lib", + # for integration tests + visibility = ["//visibility:public"], + deps = [ + ":client_connection_factory", + ":internal_listener_registry", + ], +) + +envoy_cc_library( + name = "internal_listener_registry", + srcs = [ + "internal_listener_registry.cc", + ], + hdrs = [ + "internal_listener_registry.h", + ], + deps = [ + ":thread_local_registry", + "//envoy/server:bootstrap_extension_config_interface", + "@envoy_api//envoy/extensions/bootstrap/internal_listener_registry/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "client_connection_factory", + srcs = [ + "client_connection_factory.cc", + ], + hdrs = [ + "client_connection_factory.h", + ], + deps = [ + ":thread_local_registry", + "//envoy/network:client_connection_factory", + "//envoy/network:connection_interface", + "//envoy/registry", + "//source/common/network:connection_lib", + "//source/common/network:listen_socket_lib", + "//source/extensions/io_socket/user_space:io_handle_impl_lib", + ], +) + +envoy_cc_library( + name = "thread_local_registry", + hdrs = [ + "client_connection_factory.h", + "thread_local_registry.h", + ], + deps = [ + "//envoy/network:client_connection_factory", + "//envoy/network:listener_interface", + "//envoy/thread_local:thread_local_interface", + ], +) diff --git a/source/extensions/io_socket/user_space/client_connection_factory.cc b/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.cc similarity index 90% rename from source/extensions/io_socket/user_space/client_connection_factory.cc rename to source/extensions/bootstrap/internal_listener_registry/client_connection_factory.cc index 8168fc00ce704..e0660a55ab441 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.cc +++ b/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.cc @@ -1,4 +1,4 @@ -#include "source/extensions/io_socket/user_space/client_connection_factory.h" +#include "source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h" #include "envoy/registry/registry.h" @@ -10,10 +10,10 @@ namespace Envoy { namespace Extensions { -namespace IoSocket { -namespace UserSpace { +namespace Bootstrap { +namespace InternalListenerRegistry { -ThreadLocal::TypedSlot* +ThreadLocal::TypedSlot* InternalClientConnectionFactory::registry_tls_slot_ = nullptr; Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnection( @@ -69,7 +69,7 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec REGISTER_FACTORY(InternalClientConnectionFactory, Network::ClientConnectionFactory); -} // namespace UserSpace -} // namespace IoSocket +} // namespace InternalListenerRegistry +} // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/io_socket/user_space/client_connection_factory.h b/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h similarity index 83% rename from source/extensions/io_socket/user_space/client_connection_factory.h rename to source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h index f62e383066bae..7ccc1f5b5b80e 100644 --- a/source/extensions/io_socket/user_space/client_connection_factory.h +++ b/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h @@ -8,13 +8,12 @@ #include "envoy/network/connection.h" #include "source/common/common/logger.h" -#include "source/extensions/io_socket/user_space/thread_local_registry.h" +#include "source/extensions/bootstrap/internal_listener_registry/thread_local_registry.h" namespace Envoy { - namespace Extensions { -namespace IoSocket { -namespace UserSpace { +namespace Bootstrap { +namespace InternalListenerRegistry { // This factory creates the client connection to an envoy internal address. class InternalClientConnectionFactory : public Network::ClientConnectionFactory, @@ -35,10 +34,11 @@ class InternalClientConnectionFactory : public Network::ClientConnectionFactory, // Since the population and the lookup is supposed to be executed in the same worker thread, // neither need to hold a lock. // TODO(lambdai): make it friend to only bootstrap extension. - static ThreadLocal::TypedSlot* registry_tls_slot_; + static ThreadLocal::TypedSlot* + registry_tls_slot_; }; -} // namespace UserSpace -} // namespace IoSocket +} // namespace InternalListenerRegistry +} // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.cc b/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.cc similarity index 74% rename from source/extensions/io_socket/user_space/internal_listener_registry.cc rename to source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.cc index 42603abc27d63..8cc9eea19c68a 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.cc +++ b/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.cc @@ -1,14 +1,14 @@ -#include "source/extensions/io_socket/user_space/internal_listener_registry.h" +#include "source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h" #include "envoy/singleton/manager.h" #include "source/common/singleton/threadsafe_singleton.h" -#include "source/extensions/io_socket/user_space/client_connection_factory.h" +#include "source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h" namespace Envoy { namespace Extensions { -namespace IoSocket { -namespace UserSpace { +namespace Bootstrap { +namespace InternalListenerRegistry { SINGLETON_MANAGER_REGISTRATION(internal_listener_registry); @@ -33,18 +33,18 @@ void InternalListenerExtension::onServerInitialized() { ASSERT(internal_listener_registry->tls_slot_ == nullptr); tls_registry_->tls_slot_ = - ThreadLocal::TypedSlot::makeUnique( - server_context_.threadLocal()); + ThreadLocal::TypedSlot:: + makeUnique(server_context_.threadLocal()); tls_registry_->tls_slot_->set([](Event::Dispatcher&) { - return std::make_shared(); + return std::make_shared(); }); // Now the thread local registry is available. This thread local object is published to // ``InternalClientConnectionFactory``. // Note that the per silo ``ConnectionHandler`` will add internal listeners into the per silo // registry. - Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ = - tls_registry_->tls_slot_.get(); + Extensions::Bootstrap::InternalListenerRegistry::InternalClientConnectionFactory:: + registry_tls_slot_ = tls_registry_->tls_slot_.get(); } Server::BootstrapExtensionPtr InternalListenerRegistryFactory::createBootstrapExtension( @@ -55,7 +55,7 @@ Server::BootstrapExtensionPtr InternalListenerRegistryFactory::createBootstrapEx REGISTER_FACTORY(InternalListenerRegistryFactory, Server::Configuration::BootstrapExtensionFactory); -} // namespace UserSpace -} // namespace IoSocket +} // namespace InternalListenerRegistry +} // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/io_socket/user_space/internal_listener_registry.h b/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h similarity index 78% rename from source/extensions/io_socket/user_space/internal_listener_registry.h rename to source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h index ec0e59b04a1b9..2cbf69d4ffce2 100644 --- a/source/extensions/io_socket/user_space/internal_listener_registry.h +++ b/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h @@ -1,18 +1,18 @@ #pragma once #include "envoy/config/typed_config.h" -#include "envoy/extensions/io_socket/user_space/v3/internal_listener.pb.h" +#include "envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.pb.h" #include "envoy/registry/registry.h" #include "envoy/server/bootstrap_extension_config.h" -#include "source/extensions/io_socket/user_space/thread_local_registry.h" +#include "source/extensions/bootstrap/internal_listener_registry/thread_local_registry.h" #include "absl/container/flat_hash_map.h" namespace Envoy { namespace Extensions { -namespace IoSocket { -namespace UserSpace { +namespace Bootstrap { +namespace InternalListenerRegistry { // This InternalListenerRegistry implementation owns a thread local slot. class TlsInternalListenerRegistry : public Singleton::Instance, @@ -25,7 +25,9 @@ class TlsInternalListenerRegistry : public Singleton::Instance, return nullptr; } - std::unique_ptr> tls_slot_; + std::unique_ptr< + ThreadLocal::TypedSlot> + tls_slot_; }; // This extension maintains maintain the life of the ``TlsInternalListenerRegistry`` singleton. @@ -53,12 +55,12 @@ class InternalListenerRegistryFactory : public Server::Configuration::BootstrapE Server::Configuration::ServerFactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique< - envoy::extensions::io_socket::user_space::v3::InternalListenerRegistry>(); + envoy::extensions::bootstrap::internal_listener_registry::v3::InternalListenerRegistry>(); } std::string name() const override { return "envoy.bootstrap.internal_listener_registry"; }; }; -} // namespace UserSpace -} // namespace IoSocket +} // namespace InternalListenerRegistry +} // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/io_socket/user_space/thread_local_registry.h b/source/extensions/bootstrap/internal_listener_registry/thread_local_registry.h similarity index 90% rename from source/extensions/io_socket/user_space/thread_local_registry.h rename to source/extensions/bootstrap/internal_listener_registry/thread_local_registry.h index 5859b551638d0..423cab6aa7f8c 100644 --- a/source/extensions/io_socket/user_space/thread_local_registry.h +++ b/source/extensions/bootstrap/internal_listener_registry/thread_local_registry.h @@ -5,8 +5,8 @@ namespace Envoy { namespace Extensions { -namespace IoSocket { -namespace UserSpace { +namespace Bootstrap { +namespace InternalListenerRegistry { // The registry is constructed and accessed on each silo. class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, @@ -33,7 +33,7 @@ class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, // The typical instance is the ``ConnectionHandlerImpl`` on the same thread. Network::InternalListenerManager* manager_{nullptr}; }; -} // namespace UserSpace -} // namespace IoSocket +} // namespace InternalListenerRegistry +} // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 57d6fd5eacdd6..50870d440c5b8 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -262,7 +262,7 @@ EXTENSIONS = { # "envoy.io_socket.user_space": "//source/extensions/io_socket/user_space:config", - + "envoy.bootstrap.internal_listener_registry": "//source/extensions/bootstrap/internal_listener_registry:config", # # TLS peer certification validators # diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 480bae00374bc..28e68e6ba60df 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -33,6 +33,11 @@ envoy.access_loggers.wasm: - envoy.access_loggers security_posture: unknown status: alpha +envoy.bootstrap.internal_listener_registry: + categories: + - envoy.bootstrap + security_posture: unknown + status: wip envoy.bootstrap.wasm: categories: - envoy.bootstrap diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 346bf67659f57..2ebb73ce271c2 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -12,11 +12,9 @@ envoy_extension_package() envoy_cc_extension( name = "config", srcs = ["config.h"], - # for integration tests + # for integration tests and bootstrap.internal_listener_registry visibility = ["//visibility:public"], deps = [ - ":client_connection_factory", - ":internal_listener_registry", ":io_handle_impl_lib", ], ) @@ -61,50 +59,3 @@ envoy_cc_library( "//source/common/network:default_socket_interface_lib", ], ) - -envoy_cc_library( - name = "client_connection_factory", - srcs = [ - "client_connection_factory.cc", - ], - hdrs = [ - "client_connection_factory.h", - ], - deps = [ - ":io_handle_impl_lib", - ":thread_local_registry", - "//envoy/network:client_connection_factory", - "//envoy/network:connection_interface", - "//envoy/registry", - "//source/common/network:connection_lib", - "//source/common/network:listen_socket_lib", - ], -) - -envoy_cc_library( - name = "internal_listener_registry", - srcs = [ - "internal_listener_registry.cc", - ], - hdrs = [ - "internal_listener_registry.h", - ], - deps = [ - ":thread_local_registry", - "//envoy/server:bootstrap_extension_config_interface", - "@envoy_api//envoy/extensions/io_socket/user_space/v3:pkg_cc_proto", - ], -) - -envoy_cc_library( - name = "thread_local_registry", - hdrs = [ - "client_connection_factory.h", - "thread_local_registry.h", - ], - deps = [ - "//envoy/network:client_connection_factory", - "//envoy/network:listener_interface", - "//envoy/thread_local:thread_local_interface", - ], -) diff --git a/test/extensions/bootstrap/internal_listener_registry/BUILD b/test/extensions/bootstrap/internal_listener_registry/BUILD new file mode 100644 index 0000000000000..8d7630f41108d --- /dev/null +++ b/test/extensions/bootstrap/internal_listener_registry/BUILD @@ -0,0 +1,26 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "client_connection_factory_test", + srcs = ["client_connection_factory_test.cc"], + extension_names = ["envoy.bootstrap.internal_listener_registry"], + deps = [ + "//source/common/common:utility_lib", + "//source/common/network:address_lib", + "//source/extensions/bootstrap/internal_listener_registry:client_connection_factory", + "//source/extensions/io_socket/user_space:io_handle_impl_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/thread_local:thread_local_mocks", + ], +) diff --git a/test/extensions/io_socket/user_space/client_connection_factory_test.cc b/test/extensions/bootstrap/internal_listener_registry/client_connection_factory_test.cc similarity index 85% rename from test/extensions/io_socket/user_space/client_connection_factory_test.cc rename to test/extensions/bootstrap/internal_listener_registry/client_connection_factory_test.cc index ac97b7e02619d..c06fe8e0dc17a 100644 --- a/test/extensions/io_socket/user_space/client_connection_factory_test.cc +++ b/test/extensions/bootstrap/internal_listener_registry/client_connection_factory_test.cc @@ -5,7 +5,7 @@ #include "source/common/buffer/buffer_impl.h" #include "source/common/common/fancy_logger.h" #include "source/common/network/address_impl.h" -#include "source/extensions/io_socket/user_space/client_connection_factory.h" +#include "source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h" #include "source/extensions/io_socket/user_space/io_handle_impl.h" #include "test/mocks/event/mocks.h" @@ -20,8 +20,8 @@ using testing::NiceMock; namespace Envoy { namespace Extensions { -namespace IoSocket { -namespace UserSpace { +namespace Bootstrap { +namespace InternalListenerRegistry { namespace { @@ -38,28 +38,28 @@ class ClientConnectionFactoryTest : public testing::Test { ClientConnectionFactoryTest() : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")), buf_(1024) { - std::tie(io_handle_, io_handle_peer_) = IoHandleFactory::createIoHandlePair(); + std::tie(io_handle_, io_handle_peer_) = + IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); EXPECT_CALL(tls_allocator_, allocateSlot()); - tls_slot_ = ThreadLocal::TypedSlot::makeUnique( - tls_allocator_); + tls_slot_ = ThreadLocal::TypedSlot< + Bootstrap::InternalListenerRegistry::ThreadLocalRegistryImpl>::makeUnique(tls_allocator_); tls_slot_->set([r = registry_](Event::Dispatcher&) { return r; }); // TODO: restore the original value via RAII. - Extensions::IoSocket::UserSpace::InternalClientConnectionFactory::registry_tls_slot_ = + Bootstrap::InternalListenerRegistry::InternalClientConnectionFactory::registry_tls_slot_ = tls_slot_.get(); } Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; MockInternalListenerManger internal_listener_manager_; - std::shared_ptr registry_{ - std::make_shared()}; + std::shared_ptr registry_{std::make_shared()}; ThreadLocal::MockInstance tls_allocator_; - std::unique_ptr> tls_slot_; + std::unique_ptr> tls_slot_; // Owned by IoHandleImpl. NiceMock* schedulable_cb_; - std::unique_ptr io_handle_; - std::unique_ptr io_handle_peer_; + std::unique_ptr io_handle_; + std::unique_ptr io_handle_peer_; absl::FixedArray buf_; Network::Address::EnvoyInternalInstance listener_addr{"listener_internal_address"}; }; @@ -142,7 +142,7 @@ TEST_F(ClientConnectionFactoryTest, ConnectSucceeds) { server_socket->close(); } } // namespace -} // namespace UserSpace -} // namespace IoSocket +} // namespace InternalListenerRegistry +} // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/test/extensions/io_socket/user_space/BUILD b/test/extensions/io_socket/user_space/BUILD index b5cc3df4bc83d..458d6345d36da 100644 --- a/test/extensions/io_socket/user_space/BUILD +++ b/test/extensions/io_socket/user_space/BUILD @@ -58,17 +58,3 @@ envoy_extension_cc_test( "//test/test_common:network_utility_lib", ], ) - -envoy_extension_cc_test( - name = "client_connection_factory_test", - srcs = ["client_connection_factory_test.cc"], - extension_names = ["envoy.io_socket.user_space"], - deps = [ - "//source/common/common:utility_lib", - "//source/common/network:address_lib", - "//source/extensions/io_socket/user_space:client_connection_factory", - "//source/extensions/io_socket/user_space:io_handle_impl_lib", - "//test/mocks/event:event_mocks", - "//test/mocks/thread_local:thread_local_mocks", - ], -) diff --git a/test/integration/BUILD b/test/integration/BUILD index 7c00c1e1c19a2..d007c6cf71765 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1682,7 +1682,9 @@ envoy_cc_test( "//source/common/event:dispatcher_lib", "//source/common/network:connection_lib", "//source/common/network:utility_lib", + "//source/extensions/bootstrap/internal_listener_regsitry:config", "//source/extensions/filters/network/tcp_proxy:config", + "//source/extensions/io_socket/user_space:config", "//test/common/grpc:grpc_client_integration_lib", "//test/config:v2_link_hacks", "//test/integration/filters:address_restore_listener_filter_lib", @@ -1693,8 +1695,6 @@ envoy_cc_test( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - # TODO: move to extenstion - "//source/extensions/io_socket/user_space:config", ], ) From 6e74d421b75691b1fddaaf373b0424d3a882c30b Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 00:44:50 +0000 Subject: [PATCH 40/59] address doc comment Signed-off-by: Yuchen Dai --- api/envoy/config/core/v3/address.proto | 4 ++-- .../internal_listener_registry/v3/internal_listener.proto | 3 +-- .../internal_listener_registry/internal_listener_registry.h | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/api/envoy/config/core/v3/address.proto b/api/envoy/config/core/v3/address.proto index 8813ab4dffc51..3f1b6fe3dc841 100644 --- a/api/envoy/config/core/v3/address.proto +++ b/api/envoy/config/core/v3/address.proto @@ -32,8 +32,8 @@ message Pipe { } // The address represents an envoy internal listener. -// TODO(lambdai): Make this address available for listener and endpoint. -// TODO(asraa): When address available, remove workaround from test/server/server_fuzz_test.cc:30. +// [#comment: TODO(lambdai): Make this address available for listener and endpoint. +// TODO(asraa): When address available, remove workaround from test/server/server_fuzz_test.cc:30.] message EnvoyInternalAddress { oneof address_name_specifier { option (validate.required) = true; diff --git a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto index ea95087b2c039..902e6294109ae 100644 --- a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto +++ b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto @@ -26,12 +26,11 @@ option (xds.annotations.v3.file_status).work_in_progress = true; // 2. The client connection factory `network.connection.client.envoy_internal` // // This factory is implicitly instantiated by the dispatcher to establish a client connection to the above -// internal listener, +// internal listener. // // [#extension: envoy.bootstrap.internal_listener_registry] // [#not-implemented-hide:] // Configuration for internal listener registry. -// [#extension-category: envoy.bootstrap] message InternalListenerRegistry { } diff --git a/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h b/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h index 2cbf69d4ffce2..92f31e8914820 100644 --- a/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h +++ b/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h @@ -30,7 +30,7 @@ class TlsInternalListenerRegistry : public Singleton::Instance, tls_slot_; }; -// This extension maintains maintain the life of the ``TlsInternalListenerRegistry`` singleton. +// This extension maintains the life of the ``TlsInternalListenerRegistry`` singleton. // The ``TlsInternalListenerRegistry`` is functionally ready after the Envoy thread local system is // initialized. class InternalListenerExtension : public Server::BootstrapExtension { From 60564c9af07426f365daa601e6db920c26119696 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 00:52:13 +0000 Subject: [PATCH 41/59] fix Signed-off-by: Yuchen Dai --- .../extensions/bootstrap/internal_listener_registry/v3/BUILD | 5 ++++- .../internal_listener_registry/v3/internal_listener.proto | 3 ++- source/extensions/bootstrap/internal_listener_registry/BUILD | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/BUILD b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/BUILD index ee92fb652582e..ec1e778e06e5c 100644 --- a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/BUILD +++ b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/BUILD @@ -5,5 +5,8 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = [ + "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_udpa//xds/annotations/v3:pkg", + ], ) diff --git a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto index 902e6294109ae..f6f0ec6577934 100644 --- a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto +++ b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto @@ -2,9 +2,10 @@ syntax = "proto3"; package envoy.extensions.bootstrap.internal_listener_registry.v3; -import "udpa/annotations/status.proto"; import "xds/annotations/v3/status.proto"; +import "udpa/annotations/status.proto"; + option java_package = "io.envoyproxy.envoy.extensions.bootstrap.internal_listener_registry.v3"; option java_outer_classname = "InternalListenerProto"; option java_multiple_files = true; diff --git a/source/extensions/bootstrap/internal_listener_registry/BUILD b/source/extensions/bootstrap/internal_listener_registry/BUILD index 820cf4c6a256b..a2a08cc3f83af 100644 --- a/source/extensions/bootstrap/internal_listener_registry/BUILD +++ b/source/extensions/bootstrap/internal_listener_registry/BUILD @@ -10,7 +10,7 @@ licenses(["notice"]) # Apache 2 envoy_extension_package() envoy_cc_extension( - name = "lib", + name = "config", # for integration tests visibility = ["//visibility:public"], deps = [ From ebece64efdf31a3a512a2238af5efc47b689f306 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 08:49:18 +0000 Subject: [PATCH 42/59] rewrite doc Signed-off-by: Yuchen Dai --- .../http/jwt_authn/v2alpha/config.proto | 3 +- .../v3/internal_listener.proto | 37 ------------------- .../v3/internal_listener_registry.proto | 22 +++++++++++ .../filters/http/jwt_authn/v3/config.proto | 3 +- .../other_features/internal_listener.rst | 21 +++++++++++ .../other_features/other_features.rst | 1 + source/extensions/extensions_build_config.bzl | 1 + test/integration/BUILD | 2 +- 8 files changed, 48 insertions(+), 42 deletions(-) delete mode 100644 api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto create mode 100644 api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.proto create mode 100644 docs/root/configuration/other_features/internal_listener.rst diff --git a/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto b/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto index e87c9478db635..00ce562f8dfb9 100644 --- a/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto +++ b/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto @@ -391,8 +391,7 @@ message FilterStateRule { // A map of string keys to requirements. The string key is the string value // in the FilterState with the name specified in the *name* field above. - map - requires = 3; + map requires = 3; } // This is the Envoy HTTP filter config for JWT authentication. diff --git a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto deleted file mode 100644 index f6f0ec6577934..0000000000000 --- a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.proto +++ /dev/null @@ -1,37 +0,0 @@ -syntax = "proto3"; - -package envoy.extensions.bootstrap.internal_listener_registry.v3; - -import "xds/annotations/v3/status.proto"; - -import "udpa/annotations/status.proto"; - -option java_package = "io.envoyproxy.envoy.extensions.bootstrap.internal_listener_registry.v3"; -option java_outer_classname = "InternalListenerProto"; -option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/bootstrap/internal_listener_registry/v3;internal_listener_registryv3"; -option (udpa.annotations.file_status).package_version_status = ACTIVE; -option (xds.annotations.v3.file_status).work_in_progress = true; - -// [#protodoc-title: InternalListenerRegistry] -// -// This extension contains 2 major components to add a listener with -// an :ref:`envoy internal address ` -// and to create a client connection to that :ref:`listener ` -// -// 1. The bootstrap extension `envoy.bootstrap.internal_listener_registry` -// -// This bootstrap extension is required to support looking up the target listener via an -// :ref:`envoy internal address ` on each worker threads. -// -// 2. The client connection factory `network.connection.client.envoy_internal` -// -// This factory is implicitly instantiated by the dispatcher to establish a client connection to the above -// internal listener. -// -// [#extension: envoy.bootstrap.internal_listener_registry] - -// [#not-implemented-hide:] -// Configuration for internal listener registry. -message InternalListenerRegistry { -} diff --git a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.proto b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.proto new file mode 100644 index 0000000000000..bca54d85925f1 --- /dev/null +++ b/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package envoy.extensions.bootstrap.internal_listener_registry.v3; + +import "xds/annotations/v3/status.proto"; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.bootstrap.internal_listener_registry.v3"; +option java_outer_classname = "InternalListenerRegistryProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/bootstrap/internal_listener_registry/v3;internal_listener_registryv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; +option (xds.annotations.v3.file_status).work_in_progress = true; + +// [#protodoc-title: Internal Listener Registry] +// Internal Listener :ref:`overview `. +// [#extension: envoy.bootstrap.internal_listener_registry] + +// Configuration for internal listener registry. +message InternalListenerRegistry { +} diff --git a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto index 65ddd88e9df8c..64ae483c953f1 100644 --- a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto +++ b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto @@ -583,8 +583,7 @@ message FilterStateRule { // A map of string keys to requirements. The string key is the string value // in the FilterState with the name specified in the *name* field above. - map - requires = 3; + map requires = 3; } // This is the Envoy HTTP filter config for JWT authentication. diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst new file mode 100644 index 0000000000000..234178dfc704d --- /dev/null +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -0,0 +1,21 @@ +.. _config_internal_listener: + +Internal Listener +================= + +How it works +------------ + + This extension contains 2 major components to add a listener with + an :ref:`envoy internal address ` + and to create a client connection to that :ref:`listener ` + + 1. The bootstrap extension `envoy.bootstrap.internal_listener_registry` + + This bootstrap extension is required to support looking up the target listener via an + :ref:`envoy internal address ` on each worker threads. + + 2. The client connection factory `network.connection.client.envoy_internal` + + This factory is implicitly instantiated by the dispatcher to establish a client connection to the above + internal listener. diff --git a/docs/root/configuration/other_features/other_features.rst b/docs/root/configuration/other_features/other_features.rst index 21414104fedd5..c93d34e2303e0 100644 --- a/docs/root/configuration/other_features/other_features.rst +++ b/docs/root/configuration/other_features/other_features.rst @@ -4,6 +4,7 @@ Other features .. toctree:: :maxdepth: 2 + internal_listener rate_limit vcl wasm diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 50870d440c5b8..fbca796ce1104 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -263,6 +263,7 @@ EXTENSIONS = { "envoy.io_socket.user_space": "//source/extensions/io_socket/user_space:config", "envoy.bootstrap.internal_listener_registry": "//source/extensions/bootstrap/internal_listener_registry:config", + # # TLS peer certification validators # diff --git a/test/integration/BUILD b/test/integration/BUILD index d007c6cf71765..6bea428bba1bc 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1682,7 +1682,7 @@ envoy_cc_test( "//source/common/event:dispatcher_lib", "//source/common/network:connection_lib", "//source/common/network:utility_lib", - "//source/extensions/bootstrap/internal_listener_regsitry:config", + "//source/extensions/bootstrap/internal_listener_registry:config", "//source/extensions/filters/network/tcp_proxy:config", "//source/extensions/io_socket/user_space:config", "//test/common/grpc:grpc_client_integration_lib", From d8788c2a400a6fce2ee97889fd2557357db3f917 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 11:39:56 +0000 Subject: [PATCH 43/59] fix api/BUILD and doc Signed-off-by: Yuchen Dai --- api/BUILD | 1 + docs/root/configuration/other_features/internal_listener.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api/BUILD b/api/BUILD index ff4e6094f6cdb..568181a63b02e 100644 --- a/api/BUILD +++ b/api/BUILD @@ -119,6 +119,7 @@ proto_library( "//envoy/extensions/access_loggers/open_telemetry/v3:pkg", "//envoy/extensions/access_loggers/stream/v3:pkg", "//envoy/extensions/access_loggers/wasm/v3:pkg", + "//envoy/extensions/bootstrap/internal_listener_registry/v3:pkg", "//envoy/extensions/cache/simple_http_cache/v3:pkg", "//envoy/extensions/clusters/aggregate/v3:pkg", "//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg", diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst index 234178dfc704d..1c60557580b8f 100644 --- a/docs/root/configuration/other_features/internal_listener.rst +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -1,6 +1,6 @@ .. _config_internal_listener: -Internal Listener +Internal Listener ================= How it works From 3b76567b1a49b16df97b550a8fd2d558c3de5e06 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 11:45:54 +0000 Subject: [PATCH 44/59] again in bootstrap.rst Signed-off-by: Yuchen Dai --- docs/root/api-v3/bootstrap/bootstrap.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/root/api-v3/bootstrap/bootstrap.rst b/docs/root/api-v3/bootstrap/bootstrap.rst index 4a8381a0c90f6..55902a5197c28 100644 --- a/docs/root/api-v3/bootstrap/bootstrap.rst +++ b/docs/root/api-v3/bootstrap/bootstrap.rst @@ -10,5 +10,6 @@ Bootstrap ../config/metrics/v3/metrics_service.proto ../config/overload/v3/overload.proto ../config/ratelimit/v3/rls.proto + ../extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.proto ../extensions/vcl/v3alpha/vcl_socket_interface.proto ../extensions/wasm/v3/wasm.proto From b600a4576aa7859102fd3451698acc7a517762ad Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 15:50:06 +0000 Subject: [PATCH 45/59] revert unrelated jwt config Signed-off-by: Yuchen Dai --- api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto | 3 ++- api/envoy/extensions/filters/http/jwt_authn/v3/config.proto | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto b/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto index 00ce562f8dfb9..e87c9478db635 100644 --- a/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto +++ b/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto @@ -391,7 +391,8 @@ message FilterStateRule { // A map of string keys to requirements. The string key is the string value // in the FilterState with the name specified in the *name* field above. - map requires = 3; + map + requires = 3; } // This is the Envoy HTTP filter config for JWT authentication. diff --git a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto index 64ae483c953f1..65ddd88e9df8c 100644 --- a/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto +++ b/api/envoy/extensions/filters/http/jwt_authn/v3/config.proto @@ -583,7 +583,8 @@ message FilterStateRule { // A map of string keys to requirements. The string key is the string value // in the FilterState with the name specified in the *name* field above. - map requires = 3; + map + requires = 3; } // This is the Envoy HTTP filter config for JWT authentication. From 812d19db5bfc667bc02fdbdd3af65771574979bd Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 17:02:00 +0000 Subject: [PATCH 46/59] fix include Signed-off-by: Yuchen Dai --- .../internal_listener_registry/internal_listener_registry.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h b/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h index 92f31e8914820..52365c2ea0935 100644 --- a/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h +++ b/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h @@ -1,7 +1,7 @@ #pragma once #include "envoy/config/typed_config.h" -#include "envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener.pb.h" +#include "envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.pb.h" #include "envoy/registry/registry.h" #include "envoy/server/bootstrap_extension_config.h" From 77726ce60353878161a34da75045fb1032f3a3e6 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 18:08:13 +0000 Subject: [PATCH 47/59] update test Signed-off-by: Yuchen Dai --- test/integration/internal_listener_integration_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/internal_listener_integration_test.cc b/test/integration/internal_listener_integration_test.cc index 9f223d35a751d..62f7697819edb 100644 --- a/test/integration/internal_listener_integration_test.cc +++ b/test/integration/internal_listener_integration_test.cc @@ -35,7 +35,7 @@ class InternalListenerIntegrationTest : public testing::TestWithParam Date: Wed, 2 Mar 2022 21:20:42 +0000 Subject: [PATCH 48/59] improve coverage Signed-off-by: Yuchen Dai --- .../client_connection_factory.h | 1 - .../internal_listener_registry.h | 4 +++ .../internal_listener_registry/BUILD | 10 ++++++++ .../client_connection_factory_test.cc | 25 ++++++++++++++++++- 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h b/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h index 7ccc1f5b5b80e..bab818f479c55 100644 --- a/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h +++ b/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h @@ -19,7 +19,6 @@ namespace InternalListenerRegistry { class InternalClientConnectionFactory : public Network::ClientConnectionFactory, Logger::Loggable { public: - ~InternalClientConnectionFactory() override = default; std::string name() const override { return "envoy_internal"; } Network::ClientConnectionPtr createClientConnection(Event::Dispatcher& dispatcher, diff --git a/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h b/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h index 52365c2ea0935..8a450bff5ba18 100644 --- a/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h +++ b/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h @@ -19,6 +19,10 @@ class TlsInternalListenerRegistry : public Singleton::Instance, public Network::InternalListenerRegistry { public: Network::LocalInternalListenerRegistry* getLocalRegistry() override { + // The tls slot may not initialized yet. This can happen in production when Envoy is starting. + if (!tls_slot_) { + return nullptr; + } if (auto opt = tls_slot_->get(); opt.has_value()) { return &opt.value().get(); } diff --git a/test/extensions/bootstrap/internal_listener_registry/BUILD b/test/extensions/bootstrap/internal_listener_registry/BUILD index 8d7630f41108d..5caea3ce5a6d2 100644 --- a/test/extensions/bootstrap/internal_listener_registry/BUILD +++ b/test/extensions/bootstrap/internal_listener_registry/BUILD @@ -24,3 +24,13 @@ envoy_extension_cc_test( "//test/mocks/thread_local:thread_local_mocks", ], ) + +envoy_extension_cc_test( + name = "internal_listener_registry_test", + srcs = ["internal_listener_registry_test.cc"], + extension_names = ["envoy.bootstrap.internal_listener_registry"], + deps = [ + "//source/extensions/bootstrap/internal_listener_registry:config", + "//test/mocks/thread_local:thread_local_mocks", + ], +) diff --git a/test/extensions/bootstrap/internal_listener_registry/client_connection_factory_test.cc b/test/extensions/bootstrap/internal_listener_registry/client_connection_factory_test.cc index c06fe8e0dc17a..82525f6d0788b 100644 --- a/test/extensions/bootstrap/internal_listener_registry/client_connection_factory_test.cc +++ b/test/extensions/bootstrap/internal_listener_registry/client_connection_factory_test.cc @@ -41,6 +41,8 @@ class ClientConnectionFactoryTest : public testing::Test { std::tie(io_handle_, io_handle_peer_) = IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); EXPECT_CALL(tls_allocator_, allocateSlot()); + } + void setupTlsSlot() { tls_slot_ = ThreadLocal::TypedSlot< Bootstrap::InternalListenerRegistry::ThreadLocalRegistryImpl>::makeUnique(tls_allocator_); tls_slot_->set([r = registry_](Event::Dispatcher&) { return r; }); @@ -48,7 +50,6 @@ class ClientConnectionFactoryTest : public testing::Test { Bootstrap::InternalListenerRegistry::InternalClientConnectionFactory::registry_tls_slot_ = tls_slot_.get(); } - Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; MockInternalListenerManger internal_listener_manager_; @@ -70,7 +71,27 @@ class MockInternalListener : public Network::InternalListener { MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); }; +TEST_F(ClientConnectionFactoryTest, + ConnectFailsIfInternalConnectionThreadLocalRegistryIsNotPublished) { + tls_slot_ = ThreadLocal::TypedSlot< + Bootstrap::InternalListenerRegistry::ThreadLocalRegistryImpl>::makeUnique(tls_allocator_); + // This slot set publish a nullptr. + tls_slot_->set([](Event::Dispatcher&) { return nullptr; }); + Bootstrap::InternalListenerRegistry::InternalClientConnectionFactory::registry_tls_slot_ = + tls_slot_.get(); + auto client_conn = dispatcher_->createClientConnection( + std::make_shared(listener_addr), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + EXPECT_NE(nullptr, client_conn); + EXPECT_TRUE(client_conn->connecting()); + client_conn->connect(); + // Connect returns error immediately because TLS internal listener registry is not ready. + EXPECT_FALSE(client_conn->connecting()); + client_conn->close(Network::ConnectionCloseType::NoFlush); +} + TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalConnectionManagerNotExist) { + setupTlsSlot(); auto client_conn = dispatcher_->createClientConnection( std::make_shared(listener_addr), Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); @@ -83,6 +104,7 @@ TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalConnectionManagerNotEx } TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalListenerNotExist) { + setupTlsSlot(); registry_->setInternalListenerManager(internal_listener_manager_); EXPECT_CALL(internal_listener_manager_, findByAddress(_)) @@ -103,6 +125,7 @@ TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalListenerNotExist) { // Verify that the client connection to envoy internal address can be established. This test case // does not instantiate a server connection. The server connection is tested in internal listener. TEST_F(ClientConnectionFactoryTest, ConnectSucceeds) { + setupTlsSlot(); registry_->setInternalListenerManager(internal_listener_manager_); MockInternalListener internal_listener; Network::InternalListenerOptRef internal_listener_opt{internal_listener}; From 703bc1999640a30d55e5907311d19b5f0eda1e97 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 21:51:01 +0000 Subject: [PATCH 49/59] add missing file Signed-off-by: Yuchen Dai --- .../internal_listener_registry_test.cc | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc diff --git a/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc b/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc new file mode 100644 index 0000000000000..e17575d4f9e1f --- /dev/null +++ b/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc @@ -0,0 +1,28 @@ +#include "source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h" + +#include "test/mocks/thread_local/mocks.h" +#include "test/test_common/network_utility.h" + +#include "absl/container/fixed_array.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; + +namespace Envoy { +namespace Extensions { +namespace Bootstrap { +namespace InternalListenerRegistry { + +namespace { + +TEST(TlsInternalListenerRegistryTest, TlsSlotNotInitialized) { + TlsInternalListenerRegistry registry; + ASSERT_EQ(nullptr, registry.getLocalRegistry()); +} + +} // namespace +} // namespace InternalListenerRegistry +} // namespace Bootstrap +} // namespace Extensions +} // namespace Envoy From f2ab3b75b8a2afcfaf29fc602fcd448b8aabf68f Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 21:54:51 +0000 Subject: [PATCH 50/59] clean up header Signed-off-by: Yuchen Dai --- .../internal_listener_registry_test.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc b/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc index e17575d4f9e1f..5ddad489858e2 100644 --- a/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc +++ b/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc @@ -1,9 +1,5 @@ #include "source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h" -#include "test/mocks/thread_local/mocks.h" -#include "test/test_common/network_utility.h" - -#include "absl/container/fixed_array.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -16,6 +12,7 @@ namespace InternalListenerRegistry { namespace { +// The tls slot is not initialized yet. TEST(TlsInternalListenerRegistryTest, TlsSlotNotInitialized) { TlsInternalListenerRegistry registry; ASSERT_EQ(nullptr, registry.getLocalRegistry()); From 1ba722e511998ef9c3108e1ae36a8c193a962946 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 2 Mar 2022 23:43:14 +0000 Subject: [PATCH 51/59] clangtidy Signed-off-by: Yuchen Dai --- .../internal_listener_registry_test.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc b/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc index 5ddad489858e2..f2e8081e76ab8 100644 --- a/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc +++ b/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc @@ -3,8 +3,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::NiceMock; - namespace Envoy { namespace Extensions { namespace Bootstrap { From 63a5cf24373b2359e2e7d7e1f671ad86dff3eedf Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Sun, 6 Mar 2022 22:10:57 +0000 Subject: [PATCH 52/59] add an example envoy static config and real world use case Signed-off-by: Yuchen Dai --- .../other_features/internal_listener.rst | 97 +++++++++++++++++-- 1 file changed, 88 insertions(+), 9 deletions(-) diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst index 1c60557580b8f..1700711c8a502 100644 --- a/docs/root/configuration/other_features/internal_listener.rst +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -6,16 +6,95 @@ Internal Listener How it works ------------ - This extension contains 2 major components to add a listener with - an :ref:`envoy internal address ` - and to create a client connection to that :ref:`listener ` +This extension contains 2 major components to add a listener with +an :ref:`envoy internal address ` +and to create a client connection to that :ref:`listener ` - 1. The bootstrap extension `envoy.bootstrap.internal_listener_registry` +**envoy.bootstrap.internal_listener_registry** This bootstrap extension is required to support looking up the target listener via an +:ref:`envoy internal address ` on each worker threads. - This bootstrap extension is required to support looking up the target listener via an - :ref:`envoy internal address ` on each worker threads. +**network.connection.client.envoy_internal** It is a client connection factory. The factory is implicitly instantiated by the dispatcher to establish a client connection to an +internal listener address. - 2. The client connection factory `network.connection.client.envoy_internal` +Example config +-------------- +Below is a smallest static config that redirect tcp_proxy on port 19000 to the tcp_proxy binding to the internal address. - This factory is implicitly instantiated by the dispatcher to establish a client connection to the above - internal listener. +.. code-block:: yaml + + static_resources: + listeners: + - name: outbound_tcp_svc_19000 + address: + socket_address: + address: 0.0.0.0 + port_value: 19000 + filter_chains: + - filters: + - name: tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: bridge_internal_listener + stat_prefix: svc_tcp_proxy + - name: singleton_internal_encap + address: + envoy_internal_address: + server_listener_name: singleton_internal_encap + filter_chains: + - filters: + - name: tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: singleton_internal_encap + stat_prefix: encap_tcp_proxy + clusters: + - name: bridge_internal_listener + connect_timeout: 3600s + type: STATIC + load_assignment: + cluster_name: "bridge_internal_listener" + endpoints: + - lb_endpoints: + - endpoint: + address: + envoy_internal_address: + server_listener_name: singleton_internal_encap + transport_socket: + name: envoy.transport_sockets.raw_buffer + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer + - name: singleton_internal_encap + connect_timeout: 3600s + type: STATIC + load_assignment: + cluster_name: "singleton_internal_encap" + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + bootstrap_extensions: + - name: envoy.bootstrap.internal_listener_registry + typed_config: + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener_registry.v3.InternalListenerRegistry" + layered_runtime: + layers: + - name: enable_internal_address + static_layer: + envoy.reloadable_features.internal_address: true + + +Real world use case +------------------- + +**Encap http GET requests in a HTTP CONNECT request** Currently Envoy :ref:`HttpConnectionManager ` +cannot proxy a GET request in an upstream HTTP CONNECT request. This requirement +can be acomplished by setting up the upstream endpoint of HttpConnectionManager to the internal listener address. +Meanwhile, another internal listener binding to the above listener address includes a tcp_proxy with :ref:`tunneling_config `. + +**Decap the CONNECT requests** There are some complicated GET-in-CONNECT requests across services or edges. +In order to proxy the GET request within Envoy, two layer of :ref:`HttpConnectionManager ` +is demanded. The first HttpConnectionManager layer extract the TCP stream from a CONNECT request and redirect the TCP stream to the second +HttpConnectionManager layer to parse the common GET requests. From 6f8a9b2aafe33526368705df3cf6579cc646a3e8 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Sun, 6 Mar 2022 22:40:05 +0000 Subject: [PATCH 53/59] format Signed-off-by: Yuchen Dai --- docs/root/configuration/other_features/internal_listener.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst index 1700711c8a502..64c53e80e11f7 100644 --- a/docs/root/configuration/other_features/internal_listener.rst +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -94,7 +94,7 @@ cannot proxy a GET request in an upstream HTTP CONNECT request. This requirement can be acomplished by setting up the upstream endpoint of HttpConnectionManager to the internal listener address. Meanwhile, another internal listener binding to the above listener address includes a tcp_proxy with :ref:`tunneling_config `. -**Decap the CONNECT requests** There are some complicated GET-in-CONNECT requests across services or edges. +**Decap the CONNECT requests** There are some complicated GET-in-CONNECT requests across services or edges. In order to proxy the GET request within Envoy, two layer of :ref:`HttpConnectionManager ` is demanded. The first HttpConnectionManager layer extract the TCP stream from a CONNECT request and redirect the TCP stream to the second HttpConnectionManager layer to parse the common GET requests. From fb1305b8b3e4205f71cd66e4285059dd1db5d511 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 9 Mar 2022 06:17:07 +0000 Subject: [PATCH 54/59] renaming Signed-off-by: Yuchen Dai --- CODEOWNERS | 2 +- api/BUILD | 2 +- .../v3/BUILD | 0 .../v3/internal_listener.proto} | 16 +++++------ api/versioning/BUILD | 2 +- docs/root/api-v3/bootstrap/bootstrap.rst | 2 +- .../other_features/internal_listener.rst | 6 ++-- .../BUILD | 3 +- .../client_connection_factory.cc | 8 +++--- .../client_connection_factory.h | 8 +++--- .../internal_listener_registry.cc | 28 +++++++++---------- .../internal_listener_registry.h | 18 ++++++------ .../thread_local_registry.h | 4 +-- source/extensions/extensions_build_config.bzl | 2 +- source/extensions/extensions_metadata.yaml | 2 +- source/extensions/io_socket/user_space/BUILD | 2 +- .../BUILD | 8 +++--- .../client_connection_factory_test.cc | 20 +++++++------ .../internal_listener_registry_test.cc | 6 ++-- test/integration/BUILD | 2 +- .../internal_listener_integration_test.cc | 4 +-- test/server/listener_manager_impl_test.cc | 3 +- 22 files changed, 76 insertions(+), 72 deletions(-) rename api/envoy/extensions/bootstrap/{internal_listener_registry => internal_listener}/v3/BUILD (100%) rename api/envoy/extensions/bootstrap/{internal_listener_registry/v3/internal_listener_registry.proto => internal_listener/v3/internal_listener.proto} (52%) rename source/extensions/bootstrap/{internal_listener_registry => internal_listener}/BUILD (95%) rename source/extensions/bootstrap/{internal_listener_registry => internal_listener}/client_connection_factory.cc (91%) rename source/extensions/bootstrap/{internal_listener_registry => internal_listener}/client_connection_factory.h (86%) rename source/extensions/bootstrap/{internal_listener_registry => internal_listener}/internal_listener_registry.cc (64%) rename source/extensions/bootstrap/{internal_listener_registry => internal_listener}/internal_listener_registry.h (77%) rename source/extensions/bootstrap/{internal_listener_registry => internal_listener}/thread_local_registry.h (94%) rename test/extensions/bootstrap/{internal_listener_registry => internal_listener}/BUILD (72%) rename test/extensions/bootstrap/{internal_listener_registry => internal_listener}/client_connection_factory_test.cc (91%) rename test/extensions/bootstrap/{internal_listener_registry => internal_listener}/internal_listener_registry_test.cc (70%) diff --git a/CODEOWNERS b/CODEOWNERS index e4a53110c510f..d14bb06b620d8 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -172,7 +172,7 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp /*/extensions/matching/common_inputs/environment @snowp @donyu # user space socket pair, event, connection and listener /*/extensions/io_socket/user_space @lambdai @antoniovicente -/*/extensions/bootstrap/internal_listener_registry @lambdai @adisuissa +/*/extensions/bootstrap/internal_listener @lambdai @adisuissa # Default UUID4 request ID extension /*/extensions/request_id/uuid @mattklein123 @alyssawilk # HTTP header formatters diff --git a/api/BUILD b/api/BUILD index 568181a63b02e..5a8ba2bc4ad09 100644 --- a/api/BUILD +++ b/api/BUILD @@ -119,7 +119,7 @@ proto_library( "//envoy/extensions/access_loggers/open_telemetry/v3:pkg", "//envoy/extensions/access_loggers/stream/v3:pkg", "//envoy/extensions/access_loggers/wasm/v3:pkg", - "//envoy/extensions/bootstrap/internal_listener_registry/v3:pkg", + "//envoy/extensions/bootstrap/internal_listener/v3:pkg", "//envoy/extensions/cache/simple_http_cache/v3:pkg", "//envoy/extensions/clusters/aggregate/v3:pkg", "//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg", diff --git a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/BUILD b/api/envoy/extensions/bootstrap/internal_listener/v3/BUILD similarity index 100% rename from api/envoy/extensions/bootstrap/internal_listener_registry/v3/BUILD rename to api/envoy/extensions/bootstrap/internal_listener/v3/BUILD diff --git a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.proto b/api/envoy/extensions/bootstrap/internal_listener/v3/internal_listener.proto similarity index 52% rename from api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.proto rename to api/envoy/extensions/bootstrap/internal_listener/v3/internal_listener.proto index bca54d85925f1..774a36e9e5a37 100644 --- a/api/envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.proto +++ b/api/envoy/extensions/bootstrap/internal_listener/v3/internal_listener.proto @@ -1,22 +1,22 @@ syntax = "proto3"; -package envoy.extensions.bootstrap.internal_listener_registry.v3; +package envoy.extensions.bootstrap.internal_listener.v3; import "xds/annotations/v3/status.proto"; import "udpa/annotations/status.proto"; -option java_package = "io.envoyproxy.envoy.extensions.bootstrap.internal_listener_registry.v3"; -option java_outer_classname = "InternalListenerRegistryProto"; +option java_package = "io.envoyproxy.envoy.extensions.bootstrap.internal_listener.v3"; +option java_outer_classname = "InternalListenerProto"; option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/bootstrap/internal_listener_registry/v3;internal_listener_registryv3"; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/bootstrap/internal_listener/v3;internal_listenerv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; option (xds.annotations.v3.file_status).work_in_progress = true; -// [#protodoc-title: Internal Listener Registry] +// [#protodoc-title: Internal Listener] // Internal Listener :ref:`overview `. -// [#extension: envoy.bootstrap.internal_listener_registry] +// [#extension: envoy.bootstrap.internal_listener] -// Configuration for internal listener registry. -message InternalListenerRegistry { +// Configuration for internal listener. +message InternalListener { } diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 7611530daf128..7c06ca3ce9ba0 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -56,7 +56,7 @@ proto_library( "//envoy/extensions/access_loggers/open_telemetry/v3:pkg", "//envoy/extensions/access_loggers/stream/v3:pkg", "//envoy/extensions/access_loggers/wasm/v3:pkg", - "//envoy/extensions/bootstrap/internal_listener_registry/v3:pkg", + "//envoy/extensions/bootstrap/internal_listener/v3:pkg", "//envoy/extensions/cache/simple_http_cache/v3:pkg", "//envoy/extensions/clusters/aggregate/v3:pkg", "//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg", diff --git a/docs/root/api-v3/bootstrap/bootstrap.rst b/docs/root/api-v3/bootstrap/bootstrap.rst index 55902a5197c28..8082028c72817 100644 --- a/docs/root/api-v3/bootstrap/bootstrap.rst +++ b/docs/root/api-v3/bootstrap/bootstrap.rst @@ -10,6 +10,6 @@ Bootstrap ../config/metrics/v3/metrics_service.proto ../config/overload/v3/overload.proto ../config/ratelimit/v3/rls.proto - ../extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.proto + ../extensions/bootstrap/internal_listener/v3/internal_listener.proto ../extensions/vcl/v3alpha/vcl_socket_interface.proto ../extensions/wasm/v3/wasm.proto diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst index 64c53e80e11f7..796313c476e35 100644 --- a/docs/root/configuration/other_features/internal_listener.rst +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -10,7 +10,7 @@ This extension contains 2 major components to add a listener with an :ref:`envoy internal address ` and to create a client connection to that :ref:`listener ` -**envoy.bootstrap.internal_listener_registry** This bootstrap extension is required to support looking up the target listener via an +**envoy.bootstrap.internal_listener** This bootstrap extension is required to support looking up the target listener via an :ref:`envoy internal address ` on each worker threads. **network.connection.client.envoy_internal** It is a client connection factory. The factory is implicitly instantiated by the dispatcher to establish a client connection to an @@ -76,9 +76,9 @@ Below is a smallest static config that redirect tcp_proxy on port 19000 to the t address: 0.0.0.0 port_value: 19001 bootstrap_extensions: - - name: envoy.bootstrap.internal_listener_registry + - name: envoy.bootstrap.internal_listener typed_config: - "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener_registry.v3.InternalListenerRegistry" + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" layered_runtime: layers: - name: enable_internal_address diff --git a/source/extensions/bootstrap/internal_listener_registry/BUILD b/source/extensions/bootstrap/internal_listener/BUILD similarity index 95% rename from source/extensions/bootstrap/internal_listener_registry/BUILD rename to source/extensions/bootstrap/internal_listener/BUILD index a2a08cc3f83af..83f13e01c9982 100644 --- a/source/extensions/bootstrap/internal_listener_registry/BUILD +++ b/source/extensions/bootstrap/internal_listener/BUILD @@ -30,7 +30,8 @@ envoy_cc_library( deps = [ ":thread_local_registry", "//envoy/server:bootstrap_extension_config_interface", - "@envoy_api//envoy/extensions/bootstrap/internal_listener_registry/v3:pkg_cc_proto", + "//source/server:listener_manager_lib", + "@envoy_api//envoy/extensions/bootstrap/internal_listener/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.cc b/source/extensions/bootstrap/internal_listener/client_connection_factory.cc similarity index 91% rename from source/extensions/bootstrap/internal_listener_registry/client_connection_factory.cc rename to source/extensions/bootstrap/internal_listener/client_connection_factory.cc index e0660a55ab441..59e2c8b956fdb 100644 --- a/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.cc +++ b/source/extensions/bootstrap/internal_listener/client_connection_factory.cc @@ -1,4 +1,4 @@ -#include "source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h" +#include "source/extensions/bootstrap/internal_listener/client_connection_factory.h" #include "envoy/registry/registry.h" @@ -11,9 +11,9 @@ namespace Envoy { namespace Extensions { namespace Bootstrap { -namespace InternalListenerRegistry { +namespace InternalListener { -ThreadLocal::TypedSlot* +ThreadLocal::TypedSlot* InternalClientConnectionFactory::registry_tls_slot_ = nullptr; Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnection( @@ -69,7 +69,7 @@ Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnec REGISTER_FACTORY(InternalClientConnectionFactory, Network::ClientConnectionFactory); -} // namespace InternalListenerRegistry +} // namespace InternalListener } // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h b/source/extensions/bootstrap/internal_listener/client_connection_factory.h similarity index 86% rename from source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h rename to source/extensions/bootstrap/internal_listener/client_connection_factory.h index bab818f479c55..c751f11c28a3a 100644 --- a/source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h +++ b/source/extensions/bootstrap/internal_listener/client_connection_factory.h @@ -8,12 +8,12 @@ #include "envoy/network/connection.h" #include "source/common/common/logger.h" -#include "source/extensions/bootstrap/internal_listener_registry/thread_local_registry.h" +#include "source/extensions/bootstrap/internal_listener/thread_local_registry.h" namespace Envoy { namespace Extensions { namespace Bootstrap { -namespace InternalListenerRegistry { +namespace InternalListener { // This factory creates the client connection to an envoy internal address. class InternalClientConnectionFactory : public Network::ClientConnectionFactory, @@ -33,11 +33,11 @@ class InternalClientConnectionFactory : public Network::ClientConnectionFactory, // Since the population and the lookup is supposed to be executed in the same worker thread, // neither need to hold a lock. // TODO(lambdai): make it friend to only bootstrap extension. - static ThreadLocal::TypedSlot* + static ThreadLocal::TypedSlot* registry_tls_slot_; }; -} // namespace InternalListenerRegistry +} // namespace InternalListener } // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.cc b/source/extensions/bootstrap/internal_listener/internal_listener_registry.cc similarity index 64% rename from source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.cc rename to source/extensions/bootstrap/internal_listener/internal_listener_registry.cc index 8cc9eea19c68a..354da61c54fab 100644 --- a/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.cc +++ b/source/extensions/bootstrap/internal_listener/internal_listener_registry.cc @@ -1,14 +1,14 @@ -#include "source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h" +#include "source/extensions/bootstrap/internal_listener/internal_listener_registry.h" #include "envoy/singleton/manager.h" #include "source/common/singleton/threadsafe_singleton.h" -#include "source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h" +#include "source/extensions/bootstrap/internal_listener/client_connection_factory.h" namespace Envoy { namespace Extensions { namespace Bootstrap { -namespace InternalListenerRegistry { +namespace InternalListener { SINGLETON_MANAGER_REGISTRATION(internal_listener_registry); @@ -24,38 +24,38 @@ InternalListenerExtension::InternalListenerExtension( } void InternalListenerExtension::onServerInitialized() { - std::shared_ptr internal_listener_registry = + std::shared_ptr internal_listener = server_context_.singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(internal_listener_registry)); // Save it in the singleton so the listener manager can obtain during a listener config update. - ASSERT(internal_listener_registry == tls_registry_); - ASSERT(internal_listener_registry->tls_slot_ == nullptr); + ASSERT(internal_listener == tls_registry_); + ASSERT(internal_listener->tls_slot_ == nullptr); tls_registry_->tls_slot_ = - ThreadLocal::TypedSlot:: - makeUnique(server_context_.threadLocal()); + ThreadLocal::TypedSlot::makeUnique( + server_context_.threadLocal()); tls_registry_->tls_slot_->set([](Event::Dispatcher&) { - return std::make_shared(); + return std::make_shared(); }); // Now the thread local registry is available. This thread local object is published to // ``InternalClientConnectionFactory``. // Note that the per silo ``ConnectionHandler`` will add internal listeners into the per silo // registry. - Extensions::Bootstrap::InternalListenerRegistry::InternalClientConnectionFactory:: - registry_tls_slot_ = tls_registry_->tls_slot_.get(); + Extensions::Bootstrap::InternalListener::InternalClientConnectionFactory::registry_tls_slot_ = + tls_registry_->tls_slot_.get(); } -Server::BootstrapExtensionPtr InternalListenerRegistryFactory::createBootstrapExtension( +Server::BootstrapExtensionPtr InternalListenerFactory::createBootstrapExtension( [[maybe_unused]] const Protobuf::Message& config, Server::Configuration::ServerFactoryContext& context) { return std::make_unique(context); } -REGISTER_FACTORY(InternalListenerRegistryFactory, Server::Configuration::BootstrapExtensionFactory); +REGISTER_FACTORY(InternalListenerFactory, Server::Configuration::BootstrapExtensionFactory); -} // namespace InternalListenerRegistry +} // namespace InternalListener } // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h b/source/extensions/bootstrap/internal_listener/internal_listener_registry.h similarity index 77% rename from source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h rename to source/extensions/bootstrap/internal_listener/internal_listener_registry.h index 8a450bff5ba18..2daa478afdefb 100644 --- a/source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h +++ b/source/extensions/bootstrap/internal_listener/internal_listener_registry.h @@ -1,18 +1,18 @@ #pragma once #include "envoy/config/typed_config.h" -#include "envoy/extensions/bootstrap/internal_listener_registry/v3/internal_listener_registry.pb.h" +#include "envoy/extensions/bootstrap/internal_listener/v3/internal_listener.pb.h" #include "envoy/registry/registry.h" #include "envoy/server/bootstrap_extension_config.h" -#include "source/extensions/bootstrap/internal_listener_registry/thread_local_registry.h" +#include "source/extensions/bootstrap/internal_listener/thread_local_registry.h" #include "absl/container/flat_hash_map.h" namespace Envoy { namespace Extensions { namespace Bootstrap { -namespace InternalListenerRegistry { +namespace InternalListener { // This InternalListenerRegistry implementation owns a thread local slot. class TlsInternalListenerRegistry : public Singleton::Instance, @@ -29,8 +29,7 @@ class TlsInternalListenerRegistry : public Singleton::Instance, return nullptr; } - std::unique_ptr< - ThreadLocal::TypedSlot> + std::unique_ptr> tls_slot_; }; @@ -46,12 +45,13 @@ class InternalListenerExtension : public Server::BootstrapExtension { // Server::Configuration::BootstrapExtension void onServerInitialized() override; +private: Server::Configuration::ServerFactoryContext& server_context_; std::shared_ptr tls_registry_; }; // The factory creates the `InternalListenerExtension` instance when envoy starts. -class InternalListenerRegistryFactory : public Server::Configuration::BootstrapExtensionFactory { +class InternalListenerFactory : public Server::Configuration::BootstrapExtensionFactory { public: // Server::Configuration::BootstrapExtensionFactory Server::BootstrapExtensionPtr @@ -59,12 +59,12 @@ class InternalListenerRegistryFactory : public Server::Configuration::BootstrapE Server::Configuration::ServerFactoryContext& context) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique< - envoy::extensions::bootstrap::internal_listener_registry::v3::InternalListenerRegistry>(); + envoy::extensions::bootstrap::internal_listener::v3::InternalListener>(); } - std::string name() const override { return "envoy.bootstrap.internal_listener_registry"; }; + std::string name() const override { return "envoy.bootstrap.internal_listener"; }; }; -} // namespace InternalListenerRegistry +} // namespace InternalListener } // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/bootstrap/internal_listener_registry/thread_local_registry.h b/source/extensions/bootstrap/internal_listener/thread_local_registry.h similarity index 94% rename from source/extensions/bootstrap/internal_listener_registry/thread_local_registry.h rename to source/extensions/bootstrap/internal_listener/thread_local_registry.h index 423cab6aa7f8c..f2d468590f67f 100644 --- a/source/extensions/bootstrap/internal_listener_registry/thread_local_registry.h +++ b/source/extensions/bootstrap/internal_listener/thread_local_registry.h @@ -6,7 +6,7 @@ namespace Envoy { namespace Extensions { namespace Bootstrap { -namespace InternalListenerRegistry { +namespace InternalListener { // The registry is constructed and accessed on each silo. class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, @@ -33,7 +33,7 @@ class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, // The typical instance is the ``ConnectionHandlerImpl`` on the same thread. Network::InternalListenerManager* manager_{nullptr}; }; -} // namespace InternalListenerRegistry +} // namespace InternalListener } // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index fbca796ce1104..01ee3e8766045 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -262,7 +262,7 @@ EXTENSIONS = { # "envoy.io_socket.user_space": "//source/extensions/io_socket/user_space:config", - "envoy.bootstrap.internal_listener_registry": "//source/extensions/bootstrap/internal_listener_registry:config", + "envoy.bootstrap.internal_listener": "//source/extensions/bootstrap/internal_listener:config", # # TLS peer certification validators diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 28e68e6ba60df..8b05b7744ad8a 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -33,7 +33,7 @@ envoy.access_loggers.wasm: - envoy.access_loggers security_posture: unknown status: alpha -envoy.bootstrap.internal_listener_registry: +envoy.bootstrap.internal_listener: categories: - envoy.bootstrap security_posture: unknown diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 2ebb73ce271c2..2ea8a3613353c 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -12,7 +12,7 @@ envoy_extension_package() envoy_cc_extension( name = "config", srcs = ["config.h"], - # for integration tests and bootstrap.internal_listener_registry + # for integration tests and bootstrap.internal_listener visibility = ["//visibility:public"], deps = [ ":io_handle_impl_lib", diff --git a/test/extensions/bootstrap/internal_listener_registry/BUILD b/test/extensions/bootstrap/internal_listener/BUILD similarity index 72% rename from test/extensions/bootstrap/internal_listener_registry/BUILD rename to test/extensions/bootstrap/internal_listener/BUILD index 5caea3ce5a6d2..66bd3c5b1ed1e 100644 --- a/test/extensions/bootstrap/internal_listener_registry/BUILD +++ b/test/extensions/bootstrap/internal_listener/BUILD @@ -14,11 +14,11 @@ envoy_package() envoy_extension_cc_test( name = "client_connection_factory_test", srcs = ["client_connection_factory_test.cc"], - extension_names = ["envoy.bootstrap.internal_listener_registry"], + extension_names = ["envoy.bootstrap.internal_listener"], deps = [ "//source/common/common:utility_lib", "//source/common/network:address_lib", - "//source/extensions/bootstrap/internal_listener_registry:client_connection_factory", + "//source/extensions/bootstrap/internal_listener:client_connection_factory", "//source/extensions/io_socket/user_space:io_handle_impl_lib", "//test/mocks/event:event_mocks", "//test/mocks/thread_local:thread_local_mocks", @@ -28,9 +28,9 @@ envoy_extension_cc_test( envoy_extension_cc_test( name = "internal_listener_registry_test", srcs = ["internal_listener_registry_test.cc"], - extension_names = ["envoy.bootstrap.internal_listener_registry"], + extension_names = ["envoy.bootstrap.internal_listener"], deps = [ - "//source/extensions/bootstrap/internal_listener_registry:config", + "//source/extensions/bootstrap/internal_listener:config", "//test/mocks/thread_local:thread_local_mocks", ], ) diff --git a/test/extensions/bootstrap/internal_listener_registry/client_connection_factory_test.cc b/test/extensions/bootstrap/internal_listener/client_connection_factory_test.cc similarity index 91% rename from test/extensions/bootstrap/internal_listener_registry/client_connection_factory_test.cc rename to test/extensions/bootstrap/internal_listener/client_connection_factory_test.cc index 82525f6d0788b..686e6b43f50e8 100644 --- a/test/extensions/bootstrap/internal_listener_registry/client_connection_factory_test.cc +++ b/test/extensions/bootstrap/internal_listener/client_connection_factory_test.cc @@ -5,7 +5,7 @@ #include "source/common/buffer/buffer_impl.h" #include "source/common/common/fancy_logger.h" #include "source/common/network/address_impl.h" -#include "source/extensions/bootstrap/internal_listener_registry/client_connection_factory.h" +#include "source/extensions/bootstrap/internal_listener/client_connection_factory.h" #include "source/extensions/io_socket/user_space/io_handle_impl.h" #include "test/mocks/event/mocks.h" @@ -21,7 +21,7 @@ using testing::NiceMock; namespace Envoy { namespace Extensions { namespace Bootstrap { -namespace InternalListenerRegistry { +namespace InternalListener { namespace { @@ -43,11 +43,12 @@ class ClientConnectionFactoryTest : public testing::Test { EXPECT_CALL(tls_allocator_, allocateSlot()); } void setupTlsSlot() { - tls_slot_ = ThreadLocal::TypedSlot< - Bootstrap::InternalListenerRegistry::ThreadLocalRegistryImpl>::makeUnique(tls_allocator_); + tls_slot_ = + ThreadLocal::TypedSlot::makeUnique( + tls_allocator_); tls_slot_->set([r = registry_](Event::Dispatcher&) { return r; }); // TODO: restore the original value via RAII. - Bootstrap::InternalListenerRegistry::InternalClientConnectionFactory::registry_tls_slot_ = + Bootstrap::InternalListener::InternalClientConnectionFactory::registry_tls_slot_ = tls_slot_.get(); } Api::ApiPtr api_; @@ -73,11 +74,12 @@ class MockInternalListener : public Network::InternalListener { TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalConnectionThreadLocalRegistryIsNotPublished) { - tls_slot_ = ThreadLocal::TypedSlot< - Bootstrap::InternalListenerRegistry::ThreadLocalRegistryImpl>::makeUnique(tls_allocator_); + tls_slot_ = + ThreadLocal::TypedSlot::makeUnique( + tls_allocator_); // This slot set publish a nullptr. tls_slot_->set([](Event::Dispatcher&) { return nullptr; }); - Bootstrap::InternalListenerRegistry::InternalClientConnectionFactory::registry_tls_slot_ = + Bootstrap::InternalListener::InternalClientConnectionFactory::registry_tls_slot_ = tls_slot_.get(); auto client_conn = dispatcher_->createClientConnection( std::make_shared(listener_addr), @@ -165,7 +167,7 @@ TEST_F(ClientConnectionFactoryTest, ConnectSucceeds) { server_socket->close(); } } // namespace -} // namespace InternalListenerRegistry +} // namespace InternalListener } // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc b/test/extensions/bootstrap/internal_listener/internal_listener_registry_test.cc similarity index 70% rename from test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc rename to test/extensions/bootstrap/internal_listener/internal_listener_registry_test.cc index f2e8081e76ab8..11af2f3946ac6 100644 --- a/test/extensions/bootstrap/internal_listener_registry/internal_listener_registry_test.cc +++ b/test/extensions/bootstrap/internal_listener/internal_listener_registry_test.cc @@ -1,4 +1,4 @@ -#include "source/extensions/bootstrap/internal_listener_registry/internal_listener_registry.h" +#include "source/extensions/bootstrap/internal_listener/internal_listener_registry.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -6,7 +6,7 @@ namespace Envoy { namespace Extensions { namespace Bootstrap { -namespace InternalListenerRegistry { +namespace InternalListener { namespace { @@ -17,7 +17,7 @@ TEST(TlsInternalListenerRegistryTest, TlsSlotNotInitialized) { } } // namespace -} // namespace InternalListenerRegistry +} // namespace InternalListener } // namespace Bootstrap } // namespace Extensions } // namespace Envoy diff --git a/test/integration/BUILD b/test/integration/BUILD index 6bea428bba1bc..e5ee65ef289e6 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1682,7 +1682,7 @@ envoy_cc_test( "//source/common/event:dispatcher_lib", "//source/common/network:connection_lib", "//source/common/network:utility_lib", - "//source/extensions/bootstrap/internal_listener_registry:config", + "//source/extensions/bootstrap/internal_listener:config", "//source/extensions/filters/network/tcp_proxy:config", "//source/extensions/io_socket/user_space:config", "//test/common/grpc:grpc_client_integration_lib", diff --git a/test/integration/internal_listener_integration_test.cc b/test/integration/internal_listener_integration_test.cc index 62f7697819edb..20ad6cdffa59b 100644 --- a/test/integration/internal_listener_integration_test.cc +++ b/test/integration/internal_listener_integration_test.cc @@ -33,9 +33,9 @@ class InternalListenerIntegrationTest : public testing::TestWithParam Date: Thu, 10 Mar 2022 08:41:44 +0000 Subject: [PATCH 55/59] move inline config to another file, address all htuch's comments Signed-off-by: Yuchen Dai --- .../_include/internal-listener-config.yaml | 62 +++++++++++ .../other_features/internal_listener.rst | 100 +++++------------- 2 files changed, 86 insertions(+), 76 deletions(-) create mode 100644 docs/root/configuration/other_features/_include/internal-listener-config.yaml diff --git a/docs/root/configuration/other_features/_include/internal-listener-config.yaml b/docs/root/configuration/other_features/_include/internal-listener-config.yaml new file mode 100644 index 0000000000000..15d968963fc20 --- /dev/null +++ b/docs/root/configuration/other_features/_include/internal-listener-config.yaml @@ -0,0 +1,62 @@ + static_resources: + listeners: + - name: outbound_tcp_svc_19000 + address: + socket_address: + address: 0.0.0.0 + port_value: 19000 + filter_chains: + - filters: + - name: tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: bridge_internal_listener + stat_prefix: svc_tcp_proxy + - name: singleton_internal_encap + address: + envoy_internal_address: + server_listener_name: singleton_internal_encap + filter_chains: + - filters: + - name: tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: singleton_internal_encap + stat_prefix: encap_tcp_proxy + clusters: + - name: bridge_internal_listener + connect_timeout: 3600s + type: STATIC + load_assignment: + cluster_name: "bridge_internal_listener" + endpoints: + - lb_endpoints: + - endpoint: + address: + envoy_internal_address: + server_listener_name: singleton_internal_encap + transport_socket: + name: envoy.transport_sockets.raw_buffer + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer + - name: singleton_internal_encap + connect_timeout: 3600s + type: STATIC + load_assignment: + cluster_name: "singleton_internal_encap" + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + bootstrap_extensions: + - name: envoy.bootstrap.internal_listener + typed_config: + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" + layered_runtime: + layers: + - name: enable_internal_address + static_layer: + envoy.reloadable_features.internal_address: true diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst index 796313c476e35..34baefa7cfc2c 100644 --- a/docs/root/configuration/other_features/internal_listener.rst +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -10,91 +10,39 @@ This extension contains 2 major components to add a listener with an :ref:`envoy internal address ` and to create a client connection to that :ref:`listener ` -**envoy.bootstrap.internal_listener** This bootstrap extension is required to support looking up the target listener via an +envoy.bootstrap.internal_listener +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This bootstrap extension is required to support looking up the target listener via an :ref:`envoy internal address ` on each worker threads. -**network.connection.client.envoy_internal** It is a client connection factory. The factory is implicitly instantiated by the dispatcher to establish a client connection to an -internal listener address. +network.connection.client.envoy_internal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +It is a client connection factory. The factory is implicitly instantiated by the dispatcher to establish a client connection to an +internal listener address. This client connection factory is installed automatically when **envoy.bootstrap.internal_listerner** is specified. Example config -------------- Below is a smallest static config that redirect tcp_proxy on port 19000 to the tcp_proxy binding to the internal address. -.. code-block:: yaml - - static_resources: - listeners: - - name: outbound_tcp_svc_19000 - address: - socket_address: - address: 0.0.0.0 - port_value: 19000 - filter_chains: - - filters: - - name: tcp_proxy - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy - cluster: bridge_internal_listener - stat_prefix: svc_tcp_proxy - - name: singleton_internal_encap - address: - envoy_internal_address: - server_listener_name: singleton_internal_encap - filter_chains: - - filters: - - name: tcp_proxy - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy - cluster: singleton_internal_encap - stat_prefix: encap_tcp_proxy - clusters: - - name: bridge_internal_listener - connect_timeout: 3600s - type: STATIC - load_assignment: - cluster_name: "bridge_internal_listener" - endpoints: - - lb_endpoints: - - endpoint: - address: - envoy_internal_address: - server_listener_name: singleton_internal_encap - transport_socket: - name: envoy.transport_sockets.raw_buffer - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer - - name: singleton_internal_encap - connect_timeout: 3600s - type: STATIC - load_assignment: - cluster_name: "singleton_internal_encap" - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 0.0.0.0 - port_value: 19001 - bootstrap_extensions: - - name: envoy.bootstrap.internal_listener - typed_config: - "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" - layered_runtime: - layers: - - name: enable_internal_address - static_layer: - envoy.reloadable_features.internal_address: true +.. literalinclude:: _include/internal-listener-config.yaml + :language: yaml +Real world use cases +-------------------- -Real world use case -------------------- +Encap HTTP GET requests in a HTTP CONNECT request +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -**Encap http GET requests in a HTTP CONNECT request** Currently Envoy :ref:`HttpConnectionManager ` +Currently Envoy :ref:`HTTP connection manager ` cannot proxy a GET request in an upstream HTTP CONNECT request. This requirement -can be acomplished by setting up the upstream endpoint of HttpConnectionManager to the internal listener address. -Meanwhile, another internal listener binding to the above listener address includes a tcp_proxy with :ref:`tunneling_config `. +can be acomplished by setting up the upstream endpoint of HTTP connection manager to the internal listener address. +Meanwhile, another internal listener binding to the above listener address includes a tcp_proxy with :ref:`tunneling config `. + +Decap the CONNECT requests +~~~~~~~~~~~~~~~~~~~~~~~~~~ -**Decap the CONNECT requests** There are some complicated GET-in-CONNECT requests across services or edges. -In order to proxy the GET request within Envoy, two layer of :ref:`HttpConnectionManager ` -is demanded. The first HttpConnectionManager layer extract the TCP stream from a CONNECT request and redirect the TCP stream to the second -HttpConnectionManager layer to parse the common GET requests. +There are some complicated GET-in-CONNECT requests across services or edges. +In order to proxy the GET request within Envoy, two layer of :ref:`HTTP connection manager ` +is demanded. The first HHTTP connection manager layer extract the TCP stream from a CONNECT request and redirect the TCP stream to the second +HTTP connection manager layer to parse the common GET requests. From e52ebc927c5bc25bd9d2ec0ec30fb098996b3ea2 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 10 Mar 2022 16:22:20 +0000 Subject: [PATCH 56/59] format pre Signed-off-by: Yuchen Dai --- docs/root/configuration/other_features/internal_listener.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst index 34baefa7cfc2c..69a348e97313a 100644 --- a/docs/root/configuration/other_features/internal_listener.rst +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -31,7 +31,7 @@ Below is a smallest static config that redirect tcp_proxy on port 19000 to the t Real world use cases -------------------- -Encap HTTP GET requests in a HTTP CONNECT request +Encap HTTP GET requests in a HTTP CONNECT request ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently Envoy :ref:`HTTP connection manager ` From 65972358c6d0a526abc8d43c1a4da054627a677e Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 14 Mar 2022 18:57:58 +0000 Subject: [PATCH 57/59] revert the verifiable config Signed-off-by: Yuchen Dai --- .../_include/internal-listener-config.yaml | 62 ----------------- .../other_features/internal_listener.rst | 66 ++++++++++++++++++- 2 files changed, 64 insertions(+), 64 deletions(-) delete mode 100644 docs/root/configuration/other_features/_include/internal-listener-config.yaml diff --git a/docs/root/configuration/other_features/_include/internal-listener-config.yaml b/docs/root/configuration/other_features/_include/internal-listener-config.yaml deleted file mode 100644 index 15d968963fc20..0000000000000 --- a/docs/root/configuration/other_features/_include/internal-listener-config.yaml +++ /dev/null @@ -1,62 +0,0 @@ - static_resources: - listeners: - - name: outbound_tcp_svc_19000 - address: - socket_address: - address: 0.0.0.0 - port_value: 19000 - filter_chains: - - filters: - - name: tcp_proxy - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy - cluster: bridge_internal_listener - stat_prefix: svc_tcp_proxy - - name: singleton_internal_encap - address: - envoy_internal_address: - server_listener_name: singleton_internal_encap - filter_chains: - - filters: - - name: tcp_proxy - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy - cluster: singleton_internal_encap - stat_prefix: encap_tcp_proxy - clusters: - - name: bridge_internal_listener - connect_timeout: 3600s - type: STATIC - load_assignment: - cluster_name: "bridge_internal_listener" - endpoints: - - lb_endpoints: - - endpoint: - address: - envoy_internal_address: - server_listener_name: singleton_internal_encap - transport_socket: - name: envoy.transport_sockets.raw_buffer - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer - - name: singleton_internal_encap - connect_timeout: 3600s - type: STATIC - load_assignment: - cluster_name: "singleton_internal_encap" - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 0.0.0.0 - port_value: 19001 - bootstrap_extensions: - - name: envoy.bootstrap.internal_listener - typed_config: - "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" - layered_runtime: - layers: - - name: enable_internal_address - static_layer: - envoy.reloadable_features.internal_address: true diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst index 69a348e97313a..fe3c56dbc1229 100644 --- a/docs/root/configuration/other_features/internal_listener.rst +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -25,8 +25,70 @@ Example config -------------- Below is a smallest static config that redirect tcp_proxy on port 19000 to the tcp_proxy binding to the internal address. -.. literalinclude:: _include/internal-listener-config.yaml - :language: yaml +.. code-block:: yaml + + static_resources: + listeners: + - name: outbound_tcp_svc_19000 + address: + socket_address: + address: 0.0.0.0 + port_value: 19000 + filter_chains: + - filters: + - name: tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: bridge_internal_listener + stat_prefix: svc_tcp_proxy + - name: singleton_internal_encap + address: + envoy_internal_address: + server_listener_name: singleton_internal_encap + filter_chains: + - filters: + - name: tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: singleton_internal_encap + stat_prefix: encap_tcp_proxy + clusters: + - name: bridge_internal_listener + connect_timeout: 3600s + type: STATIC + load_assignment: + cluster_name: "bridge_internal_listener" + endpoints: + - lb_endpoints: + - endpoint: + address: + envoy_internal_address: + server_listener_name: singleton_internal_encap + transport_socket: + name: envoy.transport_sockets.raw_buffer + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer + - name: singleton_internal_encap + connect_timeout: 3600s + type: STATIC + load_assignment: + cluster_name: "singleton_internal_encap" + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + bootstrap_extensions: + - name: envoy.bootstrap.internal_listener + typed_config: + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" + layered_runtime: + layers: + - name: enable_internal_address + static_layer: + envoy.reloadable_features.internal_address: true Real world use cases -------------------- From fa940135e1ddd4ef63939b034cb4b0b41927c52a Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 16 Mar 2022 21:19:47 +0000 Subject: [PATCH 58/59] move internal listener integration test to extension Signed-off-by: Yuchen Dai --- .../bootstrap/internal_listener/BUILD | 29 +++++++++++++++++++ .../internal_listener_integration_test.cc | 0 test/integration/BUILD | 28 ------------------ 3 files changed, 29 insertions(+), 28 deletions(-) rename test/{integration => extensions/bootstrap/internal_listener}/internal_listener_integration_test.cc (100%) diff --git a/test/extensions/bootstrap/internal_listener/BUILD b/test/extensions/bootstrap/internal_listener/BUILD index 66bd3c5b1ed1e..6df54805e7c81 100644 --- a/test/extensions/bootstrap/internal_listener/BUILD +++ b/test/extensions/bootstrap/internal_listener/BUILD @@ -34,3 +34,32 @@ envoy_extension_cc_test( "//test/mocks/thread_local:thread_local_mocks", ], ) + +envoy_extension_cc_test( + name = "internal_listener_integration_test", + srcs = [ + "internal_listener_integration_test.cc", + ], + extension_names = ["envoy.bootstrap.internal_listener"], + deps = [ + "//source/common/config:api_version_lib", + "//source/common/event:dispatcher_includes", + "//source/common/event:dispatcher_lib", + "//source/common/network:connection_lib", + "//source/common/network:utility_lib", + "//source/extensions/bootstrap/internal_listener:config", + "//source/extensions/filters/network/tcp_proxy:config", + "//source/extensions/io_socket/user_space:config", + "//test/common/grpc:grpc_client_integration_lib", + "//test/config:v2_link_hacks", + "//test/integration:http_integration_lib", + "//test/integration/filters:address_restore_listener_filter_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:resources_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) diff --git a/test/integration/internal_listener_integration_test.cc b/test/extensions/bootstrap/internal_listener/internal_listener_integration_test.cc similarity index 100% rename from test/integration/internal_listener_integration_test.cc rename to test/extensions/bootstrap/internal_listener/internal_listener_integration_test.cc diff --git a/test/integration/BUILD b/test/integration/BUILD index e5ee65ef289e6..c56267c42a46b 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1670,34 +1670,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "internal_listener_integration_test", - srcs = [ - "internal_listener_integration_test.cc", - ], - deps = [ - ":http_integration_lib", - "//source/common/config:api_version_lib", - "//source/common/event:dispatcher_includes", - "//source/common/event:dispatcher_lib", - "//source/common/network:connection_lib", - "//source/common/network:utility_lib", - "//source/extensions/bootstrap/internal_listener:config", - "//source/extensions/filters/network/tcp_proxy:config", - "//source/extensions/io_socket/user_space:config", - "//test/common/grpc:grpc_client_integration_lib", - "//test/config:v2_link_hacks", - "//test/integration/filters:address_restore_listener_filter_lib", - "//test/test_common:network_utility_lib", - "//test/test_common:resources_lib", - "//test/test_common:utility_lib", - "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "listener_filter_integration_test", srcs = [ From 946cdb1fa34084f49a744e7a5637340833fb4f3d Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 25 Mar 2022 17:58:56 +0000 Subject: [PATCH 59/59] fix doc per phlax Signed-off-by: Yuchen Dai --- .../configuration/other_features/internal_listener.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst index fe3c56dbc1229..7f15b6a44e448 100644 --- a/docs/root/configuration/other_features/internal_listener.rst +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -7,7 +7,7 @@ How it works ------------ This extension contains 2 major components to add a listener with -an :ref:`envoy internal address ` +an :ref:`Envoy internal address ` and to create a client connection to that :ref:`listener ` envoy.bootstrap.internal_listener @@ -19,11 +19,11 @@ This bootstrap extension is required to support looking up the target listener v network.connection.client.envoy_internal ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is a client connection factory. The factory is implicitly instantiated by the dispatcher to establish a client connection to an -internal listener address. This client connection factory is installed automatically when **envoy.bootstrap.internal_listerner** is specified. +internal listener address. This client connection factory is installed automatically when ``envoy.bootstrap.internal_listener`` is specified. Example config -------------- -Below is a smallest static config that redirect tcp_proxy on port 19000 to the tcp_proxy binding to the internal address. +Below is a smallest static config that redirect TCP proxy on port 19000 to the TCP proxy binding to the internal address. .. code-block:: yaml @@ -99,7 +99,7 @@ Encap HTTP GET requests in a HTTP CONNECT request Currently Envoy :ref:`HTTP connection manager ` cannot proxy a GET request in an upstream HTTP CONNECT request. This requirement can be acomplished by setting up the upstream endpoint of HTTP connection manager to the internal listener address. -Meanwhile, another internal listener binding to the above listener address includes a tcp_proxy with :ref:`tunneling config `. +Meanwhile, another internal listener binding to the above listener address includes a TCP proxy with :ref:`tunneling config `. Decap the CONNECT requests ~~~~~~~~~~~~~~~~~~~~~~~~~~