diff --git a/CODEOWNERS b/CODEOWNERS index d6ac2496d4541..4125d9e2b6d2e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -170,8 +170,9 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp /*/extensions/matching/input_matchers/consistent_hashing @snowp @donyu # environment generic input /*/extensions/matching/common_inputs/environment @snowp @donyu -# user space socket pair and event +# user space socket pair, event, connection and listener /*/extensions/io_socket/user_space @lambdai @antoniovicente +/*/extensions/bootstrap/internal_listener @lambdai @adisuissa # Default UUID4 request ID extension /*/extensions/request_id/uuid @mattklein123 @alyssawilk # HTTP header formatters diff --git a/api/BUILD b/api/BUILD index 3587102474dda..05eb7dbb39775 100644 --- a/api/BUILD +++ b/api/BUILD @@ -120,6 +120,7 @@ proto_library( "//envoy/extensions/access_loggers/open_telemetry/v3:pkg", "//envoy/extensions/access_loggers/stream/v3:pkg", "//envoy/extensions/access_loggers/wasm/v3:pkg", + "//envoy/extensions/bootstrap/internal_listener/v3:pkg", "//envoy/extensions/cache/simple_http_cache/v3:pkg", "//envoy/extensions/clusters/aggregate/v3:pkg", "//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg", diff --git a/api/envoy/config/core/v3/address.proto b/api/envoy/config/core/v3/address.proto index df5001bdd4f24..3f1b6fe3dc841 100644 --- a/api/envoy/config/core/v3/address.proto +++ b/api/envoy/config/core/v3/address.proto @@ -31,9 +31,9 @@ message Pipe { uint32 mode = 2 [(validate.rules).uint32 = {lte: 511}]; } -// [#not-implemented-hide:] The address represents an envoy internal listener. -// TODO(lambdai): Make this address available for listener and endpoint. -// TODO(asraa): When address available, remove workaround from test/server/server_fuzz_test.cc:30. +// The address represents an envoy internal listener. +// [#comment: TODO(lambdai): Make this address available for listener and endpoint. +// TODO(asraa): When address available, remove workaround from test/server/server_fuzz_test.cc:30.] message EnvoyInternalAddress { oneof address_name_specifier { option (validate.required) = true; diff --git a/api/envoy/extensions/bootstrap/internal_listener/v3/BUILD b/api/envoy/extensions/bootstrap/internal_listener/v3/BUILD new file mode 100644 index 0000000000000..ec1e778e06e5c --- /dev/null +++ b/api/envoy/extensions/bootstrap/internal_listener/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "@com_github_cncf_udpa//udpa/annotations:pkg", + "@com_github_cncf_udpa//xds/annotations/v3:pkg", + ], +) diff --git a/api/envoy/extensions/bootstrap/internal_listener/v3/internal_listener.proto b/api/envoy/extensions/bootstrap/internal_listener/v3/internal_listener.proto new file mode 100644 index 0000000000000..774a36e9e5a37 --- /dev/null +++ b/api/envoy/extensions/bootstrap/internal_listener/v3/internal_listener.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package envoy.extensions.bootstrap.internal_listener.v3; + +import "xds/annotations/v3/status.proto"; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.bootstrap.internal_listener.v3"; +option java_outer_classname = "InternalListenerProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/bootstrap/internal_listener/v3;internal_listenerv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; +option (xds.annotations.v3.file_status).work_in_progress = true; + +// [#protodoc-title: Internal Listener] +// Internal Listener :ref:`overview `. +// [#extension: envoy.bootstrap.internal_listener] + +// Configuration for internal listener. +message InternalListener { +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 3bd0141d477db..b668639b7803e 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -57,6 +57,7 @@ proto_library( "//envoy/extensions/access_loggers/open_telemetry/v3:pkg", "//envoy/extensions/access_loggers/stream/v3:pkg", "//envoy/extensions/access_loggers/wasm/v3:pkg", + "//envoy/extensions/bootstrap/internal_listener/v3:pkg", "//envoy/extensions/cache/simple_http_cache/v3:pkg", "//envoy/extensions/clusters/aggregate/v3:pkg", "//envoy/extensions/clusters/dynamic_forward_proxy/v3:pkg", diff --git a/docs/root/api-v3/bootstrap/bootstrap.rst b/docs/root/api-v3/bootstrap/bootstrap.rst index 4a8381a0c90f6..8082028c72817 100644 --- a/docs/root/api-v3/bootstrap/bootstrap.rst +++ b/docs/root/api-v3/bootstrap/bootstrap.rst @@ -10,5 +10,6 @@ Bootstrap ../config/metrics/v3/metrics_service.proto ../config/overload/v3/overload.proto ../config/ratelimit/v3/rls.proto + ../extensions/bootstrap/internal_listener/v3/internal_listener.proto ../extensions/vcl/v3alpha/vcl_socket_interface.proto ../extensions/wasm/v3/wasm.proto diff --git a/docs/root/configuration/other_features/internal_listener.rst b/docs/root/configuration/other_features/internal_listener.rst new file mode 100644 index 0000000000000..7f15b6a44e448 --- /dev/null +++ b/docs/root/configuration/other_features/internal_listener.rst @@ -0,0 +1,110 @@ +.. _config_internal_listener: + +Internal Listener +================= + +How it works +------------ + +This extension contains 2 major components to add a listener with +an :ref:`Envoy internal address ` +and to create a client connection to that :ref:`listener ` + +envoy.bootstrap.internal_listener +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This bootstrap extension is required to support looking up the target listener via an +:ref:`envoy internal address ` on each worker threads. + +network.connection.client.envoy_internal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +It is a client connection factory. The factory is implicitly instantiated by the dispatcher to establish a client connection to an +internal listener address. This client connection factory is installed automatically when ``envoy.bootstrap.internal_listener`` is specified. + +Example config +-------------- +Below is a smallest static config that redirect TCP proxy on port 19000 to the TCP proxy binding to the internal address. + +.. code-block:: yaml + + static_resources: + listeners: + - name: outbound_tcp_svc_19000 + address: + socket_address: + address: 0.0.0.0 + port_value: 19000 + filter_chains: + - filters: + - name: tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: bridge_internal_listener + stat_prefix: svc_tcp_proxy + - name: singleton_internal_encap + address: + envoy_internal_address: + server_listener_name: singleton_internal_encap + filter_chains: + - filters: + - name: tcp_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + cluster: singleton_internal_encap + stat_prefix: encap_tcp_proxy + clusters: + - name: bridge_internal_listener + connect_timeout: 3600s + type: STATIC + load_assignment: + cluster_name: "bridge_internal_listener" + endpoints: + - lb_endpoints: + - endpoint: + address: + envoy_internal_address: + server_listener_name: singleton_internal_encap + transport_socket: + name: envoy.transport_sockets.raw_buffer + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer + - name: singleton_internal_encap + connect_timeout: 3600s + type: STATIC + load_assignment: + cluster_name: "singleton_internal_encap" + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 0.0.0.0 + port_value: 19001 + bootstrap_extensions: + - name: envoy.bootstrap.internal_listener + typed_config: + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" + layered_runtime: + layers: + - name: enable_internal_address + static_layer: + envoy.reloadable_features.internal_address: true + +Real world use cases +-------------------- + +Encap HTTP GET requests in a HTTP CONNECT request +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Currently Envoy :ref:`HTTP connection manager ` +cannot proxy a GET request in an upstream HTTP CONNECT request. This requirement +can be acomplished by setting up the upstream endpoint of HTTP connection manager to the internal listener address. +Meanwhile, another internal listener binding to the above listener address includes a TCP proxy with :ref:`tunneling config `. + +Decap the CONNECT requests +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are some complicated GET-in-CONNECT requests across services or edges. +In order to proxy the GET request within Envoy, two layer of :ref:`HTTP connection manager ` +is demanded. The first HHTTP connection manager layer extract the TCP stream from a CONNECT request and redirect the TCP stream to the second +HTTP connection manager layer to parse the common GET requests. diff --git a/docs/root/configuration/other_features/other_features.rst b/docs/root/configuration/other_features/other_features.rst index 21414104fedd5..c93d34e2303e0 100644 --- a/docs/root/configuration/other_features/other_features.rst +++ b/docs/root/configuration/other_features/other_features.rst @@ -4,6 +4,7 @@ Other features .. toctree:: :maxdepth: 2 + internal_listener rate_limit vcl wasm diff --git a/envoy/network/BUILD b/envoy/network/BUILD index dee05204c2796..adf3821b22a74 100644 --- a/envoy/network/BUILD +++ b/envoy/network/BUILD @@ -28,6 +28,18 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "client_connection_factory", + hdrs = ["client_connection_factory.h"], + deps = [ + ":address_interface", + ":connection_interface", + ":listen_socket_interface", + ":transport_socket_interface", + "//envoy/config:typed_config_interface", + ], +) + envoy_cc_library( name = "connection_handler_interface", hdrs = ["connection_handler.h"], diff --git a/envoy/network/address.h b/envoy/network/address.h index 59f1bd54e0d37..1cf2aa17ac53a 100644 --- a/envoy/network/address.h +++ b/envoy/network/address.h @@ -210,6 +210,12 @@ class Instance { */ virtual Type type() const PURE; + /** + * Return the address type in string_view. The returned type name is used to find the + * ClientConnectionFactory. + */ + virtual absl::string_view addressType() const PURE; + /** * @return SocketInterface to be used with the address. */ diff --git a/envoy/network/client_connection_factory.h b/envoy/network/client_connection_factory.h new file mode 100644 index 0000000000000..c9a69b2ca4f43 --- /dev/null +++ b/envoy/network/client_connection_factory.h @@ -0,0 +1,38 @@ +#pragma once + +#include "envoy/config/typed_config.h" +#include "envoy/network/address.h" +#include "envoy/network/connection.h" +#include "envoy/network/listen_socket.h" +#include "envoy/network/transport_socket.h" + +namespace Envoy { +namespace Network { + +// The factory to create a client connection. This factory hides the details of various remote +// address type and transport socket. +class ClientConnectionFactory : public Config::UntypedFactory { +public: + ~ClientConnectionFactory() override = default; + + // Config::UntypedFactory + std::string category() const override { return "network.connection.client"; } + + /** + * @param address The target remote address. + * @param source_address Optional source address. + * @param transport_socket The transport socket which supports this client connection. + * @param options The optional socket options. + * @return Network::ClientConnectionPtr The created connection. It's never nullptr but the return + * connection may be closed upon return. + */ + virtual Network::ClientConnectionPtr + createClientConnection(Event::Dispatcher& dispatcher, + Network::Address::InstanceConstSharedPtr address, + Network::Address::InstanceConstSharedPtr source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) PURE; +}; + +} // namespace Network +} // namespace Envoy diff --git a/envoy/network/listener.h b/envoy/network/listener.h index fac995eb3ed6f..2a4c3f97908ce 100644 --- a/envoy/network/listener.h +++ b/envoy/network/listener.h @@ -107,12 +107,21 @@ class UdpListenerConfig { using UdpListenerConfigOptRef = OptRef; +// Forward declare. +class InternalListenerRegistry; + /** * Configuration for an internal listener. */ class InternalListenerConfig { public: virtual ~InternalListenerConfig() = default; + + /** + * @return InternalListenerRegistry& The internal registry of this internal listener config. The + * registry outlives this listener config. + */ + virtual InternalListenerRegistry& internalListenerRegistry() PURE; }; using InternalListenerConfigOptRef = OptRef; @@ -472,7 +481,7 @@ class InternalListenerManager { virtual ~InternalListenerManager() = default; /** - * Return the internal listener callbacks binding the listener address. + * Return the internal listener binding the listener address. * * @param listen_address the internal address of the expected internal listener. */ @@ -483,6 +492,30 @@ class InternalListenerManager { using InternalListenerManagerOptRef = absl::optional>; +// The thread local registry. +class LocalInternalListenerRegistry { +public: + virtual ~LocalInternalListenerRegistry() = default; + + // Set the internal listener manager which maintains life of internal listeners. Called by + // connection handler. + virtual void setInternalListenerManager(InternalListenerManager& internal_listener_manager) PURE; + + // Get the internal listener manager to obtain a listener. Called by client connection factory. + virtual Network::InternalListenerManagerOptRef getInternalListenerManager() PURE; +}; + +// The central internal listener registry interface providing the thread local accessor. +class InternalListenerRegistry { +public: + virtual ~InternalListenerRegistry() = default; + + /** + * @return The thread local registry. + */ + virtual LocalInternalListenerRegistry* getLocalRegistry() PURE; +}; + /** * Handles delivering datagrams to the correct worker. */ diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 22efdc958ac26..302319d88c808 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -39,12 +39,15 @@ envoy_cc_library( "//envoy/common:scope_tracker_interface", "//envoy/common:time_interface", "//envoy/event:signal_interface", + "//envoy/network:client_connection_factory", "//envoy/network:listen_socket_interface", "//envoy/network:listener_interface", "//source/common/common:assert_lib", "//source/common/common:thread_lib", + "//source/common/config:utility_lib", "//source/common/filesystem:watcher_lib", - "//source/common/network:connection_lib", + "//source/common/network:address_lib", + "//source/common/network:default_client_connection_factory", "//source/common/network:listener_lib", "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", ], diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index e5f3b9ee4ea22..b066e3ae6bb9c 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -9,6 +9,7 @@ #include "envoy/api/api.h" #include "envoy/common/scope_tracker.h" #include "envoy/config/overload/v3/overload.pb.h" +#include "envoy/network/client_connection_factory.h" #include "envoy/network/listen_socket.h" #include "envoy/network/listener.h" @@ -16,12 +17,14 @@ #include "source/common/common/assert.h" #include "source/common/common/lock_guard.h" #include "source/common/common/thread.h" +#include "source/common/config/utility.h" #include "source/common/event/file_event_impl.h" #include "source/common/event/libevent_scheduler.h" #include "source/common/event/scaled_range_timer_manager_impl.h" #include "source/common/event/signal_impl.h" #include "source/common/event/timer_impl.h" #include "source/common/filesystem/watcher_impl.h" +#include "source/common/network/address_impl.h" #include "source/common/network/connection_impl.h" #include "source/common/network/tcp_listener_impl.h" #include "source/common/network/udp_listener_impl.h" @@ -158,8 +161,16 @@ DispatcherImpl::createClientConnection(Network::Address::InstanceConstSharedPtr Network::TransportSocketPtr&& transport_socket, const Network::ConnectionSocket::OptionsSharedPtr& options) { ASSERT(isThreadSafe()); - return std::make_unique(*this, address, source_address, - std::move(transport_socket), options); + + auto* factory = Config::Utility::getFactoryByName( + std::string(address->addressType())); + // The target address is usually offered by EDS and the EDS api should reject the unsupported + // address. + // TODO(lambdai): Return a closed connection if the factory is not found. Note that the caller + // expects a non-null connection as of today so we cannot gracefully handle unsupported address + // type. + return factory->createClientConnection(*this, address, source_address, + std::move(transport_socket), options); } FileEventPtr DispatcherImpl::createFileEvent(os_fd_t fd, FileReadyCb cb, FileTriggerType trigger, diff --git a/source/common/network/BUILD b/source/common/network/BUILD index fb8114973294a..eb2976555c631 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -82,6 +82,14 @@ envoy_cc_library( envoy_cc_library( name = "connection_lib", + deps = [ + ":connection_impl", + ":default_client_connection_factory", + ], +) + +envoy_cc_library( + name = "connection_impl", srcs = ["connection_impl.cc"], hdrs = ["connection_impl.h"], external_deps = ["abseil_optional"], @@ -117,6 +125,25 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "default_client_connection_factory", + srcs = [ + "default_client_connection_factory.cc", + ], + hdrs = [ + "default_client_connection_factory.h", + ], + deps = [ + ":address_lib", + ":connection_base_lib", + ":connection_impl", + "//envoy/network:client_connection_factory", + "//envoy/network:connection_interface", + "//envoy/network:transport_socket_interface", + "//envoy/registry", + ], +) + envoy_cc_library( name = "filter_lib", hdrs = ["filter_impl.h"], diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index cf8ddaee131f5..bd6a5467d5062 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -123,6 +123,7 @@ class Ipv4Instance : public InstanceBase { return reinterpret_cast(&ip_.ipv4_.address_); } socklen_t sockAddrLen() const override { return sizeof(sockaddr_in); } + absl::string_view addressType() const override { return "default"; } /** * Convenience function to convert an IPv4 address to canonical string format. @@ -213,6 +214,7 @@ class Ipv6Instance : public InstanceBase { return reinterpret_cast(&ip_.ipv6_.address_); } socklen_t sockAddrLen() const override { return sizeof(sockaddr_in6); } + absl::string_view addressType() const override { return "default"; } // Validate that IPv6 is supported on this platform static absl::Status validateProtocolSupported(); @@ -300,6 +302,7 @@ class PipeInstance : public InstanceBase { } return sizeof(pipe_.address_); } + absl::string_view addressType() const override { return "default"; } private: /** @@ -345,6 +348,7 @@ class EnvoyInternalInstance : public InstanceBase { // TODO(lambdai): Verify all callers accepts nullptr. const sockaddr* sockAddr() const override { return nullptr; } socklen_t sockAddrLen() const override { return 0; } + absl::string_view addressType() const override { return "envoy_internal"; } private: struct EnvoyInternalAddressImpl : public EnvoyInternalAddress { diff --git a/source/common/network/default_client_connection_factory.cc b/source/common/network/default_client_connection_factory.cc new file mode 100644 index 0000000000000..e1a65a0e81579 --- /dev/null +++ b/source/common/network/default_client_connection_factory.cc @@ -0,0 +1,24 @@ +#include "source/common/network/default_client_connection_factory.h" + +#include "envoy/registry/registry.h" + +#include "source/common/network/address_impl.h" +#include "source/common/network/connection_impl.h" + +namespace Envoy { + +namespace Network { + +Network::ClientConnectionPtr DefaultClientConnectionFactory::createClientConnection( + Event::Dispatcher& dispatcher, Network::Address::InstanceConstSharedPtr address, + Network::Address::InstanceConstSharedPtr source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) { + ASSERT(address->ip() || address->pipe()); + return std::make_unique(dispatcher, address, source_address, + std::move(transport_socket), options); +} +REGISTER_FACTORY(DefaultClientConnectionFactory, Network::ClientConnectionFactory); + +} // namespace Network +} // namespace Envoy diff --git a/source/common/network/default_client_connection_factory.h b/source/common/network/default_client_connection_factory.h new file mode 100644 index 0000000000000..6658e2d3093c0 --- /dev/null +++ b/source/common/network/default_client_connection_factory.h @@ -0,0 +1,32 @@ +#pragma once + +#include "envoy/common/pure.h" +#include "envoy/network/client_connection_factory.h" +#include "envoy/network/connection.h" + +namespace Envoy { + +namespace Network { + +/** + * This client connection factory handles the connection if the remote address type is either ip or + * pipe. + */ +class DefaultClientConnectionFactory : public ClientConnectionFactory { +public: + ~DefaultClientConnectionFactory() override = default; + + // Config::UntypedFactory + std::string name() const override { return "default"; } + + // Network::ClientConnectionFactory + Network::ClientConnectionPtr + createClientConnection(Event::Dispatcher& dispatcher, + Network::Address::InstanceConstSharedPtr address, + Network::Address::InstanceConstSharedPtr source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) override; +}; + +} // namespace Network +} // namespace Envoy diff --git a/source/extensions/bootstrap/internal_listener/BUILD b/source/extensions/bootstrap/internal_listener/BUILD new file mode 100644 index 0000000000000..83f13e01c9982 --- /dev/null +++ b/source/extensions/bootstrap/internal_listener/BUILD @@ -0,0 +1,68 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "config", + # for integration tests + visibility = ["//visibility:public"], + deps = [ + ":client_connection_factory", + ":internal_listener_registry", + ], +) + +envoy_cc_library( + name = "internal_listener_registry", + srcs = [ + "internal_listener_registry.cc", + ], + hdrs = [ + "internal_listener_registry.h", + ], + deps = [ + ":thread_local_registry", + "//envoy/server:bootstrap_extension_config_interface", + "//source/server:listener_manager_lib", + "@envoy_api//envoy/extensions/bootstrap/internal_listener/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "client_connection_factory", + srcs = [ + "client_connection_factory.cc", + ], + hdrs = [ + "client_connection_factory.h", + ], + deps = [ + ":thread_local_registry", + "//envoy/network:client_connection_factory", + "//envoy/network:connection_interface", + "//envoy/registry", + "//source/common/network:connection_lib", + "//source/common/network:listen_socket_lib", + "//source/extensions/io_socket/user_space:io_handle_impl_lib", + ], +) + +envoy_cc_library( + name = "thread_local_registry", + hdrs = [ + "client_connection_factory.h", + "thread_local_registry.h", + ], + deps = [ + "//envoy/network:client_connection_factory", + "//envoy/network:listener_interface", + "//envoy/thread_local:thread_local_interface", + ], +) diff --git a/source/extensions/bootstrap/internal_listener/client_connection_factory.cc b/source/extensions/bootstrap/internal_listener/client_connection_factory.cc new file mode 100644 index 0000000000000..59e2c8b956fdb --- /dev/null +++ b/source/extensions/bootstrap/internal_listener/client_connection_factory.cc @@ -0,0 +1,75 @@ +#include "source/extensions/bootstrap/internal_listener/client_connection_factory.h" + +#include "envoy/registry/registry.h" + +#include "source/common/network/address_impl.h" +#include "source/common/network/connection_impl.h" +#include "source/common/network/listen_socket_impl.h" +#include "source/extensions/io_socket/user_space/io_handle_impl.h" + +namespace Envoy { + +namespace Extensions { +namespace Bootstrap { +namespace InternalListener { + +ThreadLocal::TypedSlot* + InternalClientConnectionFactory::registry_tls_slot_ = nullptr; + +Network::ClientConnectionPtr InternalClientConnectionFactory::createClientConnection( + Event::Dispatcher& dispatcher, Network::Address::InstanceConstSharedPtr address, + Network::Address::InstanceConstSharedPtr source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) { + // OS does not fill the address automatically so a pivotal address is populated. + // TODO(lambdai): provide option to fill the downstream remote address here. + if (source_address == nullptr) { + source_address = + std::make_shared("internal_client_address"); + } + + auto [io_handle_client, io_handle_server] = + // 1MB is the default connection buffer size. + // TODO(lambdai): export as configuration. + Extensions::IoSocket::UserSpace::IoHandleFactory::createBufferLimitedIoHandlePair(1024 * + 1024); + + auto client_conn = std::make_unique( + dispatcher, + std::make_unique(std::move(io_handle_client), source_address, + address), + source_address, std::move(transport_socket), options); + + if (registry_tls_slot_ == nullptr || !registry_tls_slot_->get().has_value()) { + ENVOY_LOG_MISC(debug, + "server has not initialized internal listener registry, close the connection"); + io_handle_server->close(); + return client_conn; + } + + // It's either in the main thread or the worker is not yet started. + auto internal_listener_manager = registry_tls_slot_->get()->getInternalListenerManager(); + if (!internal_listener_manager.has_value()) { + io_handle_server->close(); + return client_conn; + } + + // The requested internal listener is not available yet. + auto internal_listener = internal_listener_manager.value().get().findByAddress(address); + if (!internal_listener.has_value()) { + io_handle_server->close(); + return client_conn; + } + + auto accepted_socket = std::make_unique(std::move(io_handle_server), + address, source_address); + internal_listener->onAccept(std::move(accepted_socket)); + return client_conn; +} + +REGISTER_FACTORY(InternalClientConnectionFactory, Network::ClientConnectionFactory); + +} // namespace InternalListener +} // namespace Bootstrap +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/bootstrap/internal_listener/client_connection_factory.h b/source/extensions/bootstrap/internal_listener/client_connection_factory.h new file mode 100644 index 0000000000000..c751f11c28a3a --- /dev/null +++ b/source/extensions/bootstrap/internal_listener/client_connection_factory.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include + +#include "envoy/common/pure.h" +#include "envoy/network/client_connection_factory.h" +#include "envoy/network/connection.h" + +#include "source/common/common/logger.h" +#include "source/extensions/bootstrap/internal_listener/thread_local_registry.h" + +namespace Envoy { +namespace Extensions { +namespace Bootstrap { +namespace InternalListener { + +// This factory creates the client connection to an envoy internal address. +class InternalClientConnectionFactory : public Network::ClientConnectionFactory, + Logger::Loggable { +public: + std::string name() const override { return "envoy_internal"; } + Network::ClientConnectionPtr + createClientConnection(Event::Dispatcher& dispatcher, + Network::Address::InstanceConstSharedPtr address, + Network::Address::InstanceConstSharedPtr source_address, + Network::TransportSocketPtr&& transport_socket, + const Network::ConnectionSocket::OptionsSharedPtr& options) override; + // The slot is owned by the internal listener registry extension. Once that extension is + // initialized, this slot is available. The ClientConnectionFactory has two potential user cases. + // 1. The per worker thread connection handler populates the per worker listener registry. + // 2. A envoy thread local cluster lookup the per thread internal listener by listener name. + // Since the population and the lookup is supposed to be executed in the same worker thread, + // neither need to hold a lock. + // TODO(lambdai): make it friend to only bootstrap extension. + static ThreadLocal::TypedSlot* + registry_tls_slot_; +}; + +} // namespace InternalListener +} // namespace Bootstrap +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/bootstrap/internal_listener/internal_listener_registry.cc b/source/extensions/bootstrap/internal_listener/internal_listener_registry.cc new file mode 100644 index 0000000000000..354da61c54fab --- /dev/null +++ b/source/extensions/bootstrap/internal_listener/internal_listener_registry.cc @@ -0,0 +1,61 @@ +#include "source/extensions/bootstrap/internal_listener/internal_listener_registry.h" + +#include "envoy/singleton/manager.h" + +#include "source/common/singleton/threadsafe_singleton.h" +#include "source/extensions/bootstrap/internal_listener/client_connection_factory.h" + +namespace Envoy { +namespace Extensions { +namespace Bootstrap { +namespace InternalListener { + +SINGLETON_MANAGER_REGISTRATION(internal_listener_registry); + +InternalListenerExtension::InternalListenerExtension( + Server::Configuration::ServerFactoryContext& server_context) + : server_context_(server_context), + tls_registry_(std::make_shared()) { + // Initialize this singleton before the listener manager potentially load a static internal + // listener. + server_context_.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(internal_listener_registry), + [registry = tls_registry_]() { return registry; }); +} + +void InternalListenerExtension::onServerInitialized() { + std::shared_ptr internal_listener = + server_context_.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(internal_listener_registry)); + + // Save it in the singleton so the listener manager can obtain during a listener config update. + ASSERT(internal_listener == tls_registry_); + ASSERT(internal_listener->tls_slot_ == nullptr); + + tls_registry_->tls_slot_ = + ThreadLocal::TypedSlot::makeUnique( + server_context_.threadLocal()); + tls_registry_->tls_slot_->set([](Event::Dispatcher&) { + return std::make_shared(); + }); + + // Now the thread local registry is available. This thread local object is published to + // ``InternalClientConnectionFactory``. + // Note that the per silo ``ConnectionHandler`` will add internal listeners into the per silo + // registry. + Extensions::Bootstrap::InternalListener::InternalClientConnectionFactory::registry_tls_slot_ = + tls_registry_->tls_slot_.get(); +} + +Server::BootstrapExtensionPtr InternalListenerFactory::createBootstrapExtension( + [[maybe_unused]] const Protobuf::Message& config, + Server::Configuration::ServerFactoryContext& context) { + return std::make_unique(context); +} + +REGISTER_FACTORY(InternalListenerFactory, Server::Configuration::BootstrapExtensionFactory); + +} // namespace InternalListener +} // namespace Bootstrap +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/bootstrap/internal_listener/internal_listener_registry.h b/source/extensions/bootstrap/internal_listener/internal_listener_registry.h new file mode 100644 index 0000000000000..2daa478afdefb --- /dev/null +++ b/source/extensions/bootstrap/internal_listener/internal_listener_registry.h @@ -0,0 +1,70 @@ +#pragma once + +#include "envoy/config/typed_config.h" +#include "envoy/extensions/bootstrap/internal_listener/v3/internal_listener.pb.h" +#include "envoy/registry/registry.h" +#include "envoy/server/bootstrap_extension_config.h" + +#include "source/extensions/bootstrap/internal_listener/thread_local_registry.h" + +#include "absl/container/flat_hash_map.h" + +namespace Envoy { +namespace Extensions { +namespace Bootstrap { +namespace InternalListener { + +// This InternalListenerRegistry implementation owns a thread local slot. +class TlsInternalListenerRegistry : public Singleton::Instance, + public Network::InternalListenerRegistry { +public: + Network::LocalInternalListenerRegistry* getLocalRegistry() override { + // The tls slot may not initialized yet. This can happen in production when Envoy is starting. + if (!tls_slot_) { + return nullptr; + } + if (auto opt = tls_slot_->get(); opt.has_value()) { + return &opt.value().get(); + } + return nullptr; + } + + std::unique_ptr> + tls_slot_; +}; + +// This extension maintains the life of the ``TlsInternalListenerRegistry`` singleton. +// The ``TlsInternalListenerRegistry`` is functionally ready after the Envoy thread local system is +// initialized. +class InternalListenerExtension : public Server::BootstrapExtension { +public: + explicit InternalListenerExtension(Server::Configuration::ServerFactoryContext& server_context); + + ~InternalListenerExtension() override = default; + + // Server::Configuration::BootstrapExtension + void onServerInitialized() override; + +private: + Server::Configuration::ServerFactoryContext& server_context_; + std::shared_ptr tls_registry_; +}; + +// The factory creates the `InternalListenerExtension` instance when envoy starts. +class InternalListenerFactory : public Server::Configuration::BootstrapExtensionFactory { +public: + // Server::Configuration::BootstrapExtensionFactory + Server::BootstrapExtensionPtr + createBootstrapExtension(const Protobuf::Message& config, + Server::Configuration::ServerFactoryContext& context) override; + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique< + envoy::extensions::bootstrap::internal_listener::v3::InternalListener>(); + } + std::string name() const override { return "envoy.bootstrap.internal_listener"; }; +}; + +} // namespace InternalListener +} // namespace Bootstrap +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/bootstrap/internal_listener/thread_local_registry.h b/source/extensions/bootstrap/internal_listener/thread_local_registry.h new file mode 100644 index 0000000000000..f2d468590f67f --- /dev/null +++ b/source/extensions/bootstrap/internal_listener/thread_local_registry.h @@ -0,0 +1,39 @@ +#pragma once + +#include "envoy/network/listener.h" +#include "envoy/thread_local/thread_local_object.h" + +namespace Envoy { +namespace Extensions { +namespace Bootstrap { +namespace InternalListener { + +// The registry is constructed and accessed on each silo. +class ThreadLocalRegistryImpl : public ThreadLocal::ThreadLocalObject, + public Network::LocalInternalListenerRegistry { +public: + ThreadLocalRegistryImpl() = default; + + // Network::LocalInternalListenerRegistry + void + setInternalListenerManager(Network::InternalListenerManager& internal_listener_manager) override { + manager_ = &internal_listener_manager; + } + + Network::InternalListenerManagerOptRef getInternalListenerManager() override { + if (manager_ == nullptr) { + // The internal listener manager is published to this registry when the first internal + // listener is added through LDS. Return null prior to this moment. + return Network::InternalListenerManagerOptRef(); + } + return Network::InternalListenerManagerOptRef(*manager_); + } + +private: + // The typical instance is the ``ConnectionHandlerImpl`` on the same thread. + Network::InternalListenerManager* manager_{nullptr}; +}; +} // namespace InternalListener +} // namespace Bootstrap +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 9bf7950e20ed0..0dcdda93ace89 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -268,6 +268,7 @@ EXTENSIONS = { # "envoy.io_socket.user_space": "//source/extensions/io_socket/user_space:config", + "envoy.bootstrap.internal_listener": "//source/extensions/bootstrap/internal_listener:config", # # TLS peer certification validators diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 21722c02f5198..af43c81ed081b 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -33,6 +33,11 @@ envoy.access_loggers.wasm: - envoy.access_loggers security_posture: unknown status: alpha +envoy.bootstrap.internal_listener: + categories: + - envoy.bootstrap + security_posture: unknown + status: wip envoy.bootstrap.wasm: categories: - envoy.bootstrap diff --git a/source/extensions/io_socket/user_space/BUILD b/source/extensions/io_socket/user_space/BUILD index 01a5948f4c9e4..2ea8a3613353c 100644 --- a/source/extensions/io_socket/user_space/BUILD +++ b/source/extensions/io_socket/user_space/BUILD @@ -12,6 +12,8 @@ envoy_extension_package() envoy_cc_extension( name = "config", srcs = ["config.h"], + # for integration tests and bootstrap.internal_listener + visibility = ["//visibility:public"], deps = [ ":io_handle_impl_lib", ], diff --git a/source/extensions/io_socket/user_space/io_handle_impl.h b/source/extensions/io_socket/user_space/io_handle_impl.h index 5b0bdf1daa319..12c91a941b7f4 100644 --- a/source/extensions/io_socket/user_space/io_handle_impl.h +++ b/source/extensions/io_socket/user_space/io_handle_impl.h @@ -186,6 +186,17 @@ class IoHandleFactory { p.second->setPeerHandle(p.first.get()); return p; } + static std::pair + createBufferLimitedIoHandlePair(uint32_t buffer_size) { + auto p = std::pair{new IoHandleImpl(), new IoHandleImpl()}; + // This buffer watermark setting emulates the OS socket buffer parameter + // `/proc/sys/net/ipv4/tcp_{r,w}mem`. + p.first->setWatermarks(buffer_size); + p.second->setWatermarks(buffer_size); + p.first->setPeerHandle(p.second.get()); + p.second->setPeerHandle(p.first.get()); + return p; + } }; } // namespace UserSpace } // namespace IoSocket diff --git a/source/server/BUILD b/source/server/BUILD index 0a7eec52f7d05..a408ed0905ba4 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -68,6 +68,8 @@ envoy_cc_library( ":active_tcp_listener", ":active_udp_listener", ":connection_handler_impl", + "//envoy/network:client_connection_factory", + "//source/common/network:address_lib", ], ) diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index 91a8d2a88f7c9..94699da0ed53a 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -42,6 +42,14 @@ void ConnectionHandlerImpl::addListener(absl::optional overridden_list auto details = std::make_shared(); if (config.internalListenerConfig().has_value()) { + // Ensure the this ConnectionHandlerImpl link to the thread local registry. Ideally this step + // should be done only once. However, an extra phase and interface is overkill. + Network::InternalListenerRegistry& internal_listener_registry = + config.internalListenerConfig()->internalListenerRegistry(); + Network::LocalInternalListenerRegistry* local_registry = + internal_listener_registry.getLocalRegistry(); + RELEASE_ASSERT(local_registry != nullptr, "Failed to get local internal listener registry."); + local_registry->setInternalListenerManager(*this); if (overridden_listener.has_value()) { if (auto iter = listener_map_by_tag_.find(overridden_listener.value()); iter != listener_map_by_tag_.end()) { diff --git a/source/server/listener_impl.cc b/source/server/listener_impl.cc index de0210d9cd2f1..9fdc5f84d8870 100644 --- a/source/server/listener_impl.cc +++ b/source/server/listener_impl.cc @@ -8,6 +8,7 @@ #include "envoy/registry/registry.h" #include "envoy/server/options.h" #include "envoy/server/transport_socket_config.h" +#include "envoy/singleton/manager.h" #include "envoy/stats/scope.h" #include "source/common/access_log/access_log_impl.h" @@ -484,7 +485,6 @@ void ListenerImpl::buildAccessLog() { void ListenerImpl::buildInternalListener() { if (config_.address().has_envoy_internal_address()) { - internal_listener_config_ = std::make_unique(); if (config_.has_api_listener()) { throw EnvoyException( fmt::format("error adding listener '{}': internal address cannot be used in api listener", @@ -505,6 +505,16 @@ void ListenerImpl::buildInternalListener() { throw EnvoyException(fmt::format("error adding listener '{}': does not support socket option", address_->asString())); } + std::shared_ptr internal_listener_registry = + parent_.server_.singletonManager().getTyped( + "internal_listener_registry_singleton"); + if (internal_listener_registry == nullptr) { + throw EnvoyException( + fmt::format("error adding listener '{}': internal listener registry is not initialized.", + address_->asString())); + } + internal_listener_config_ = + std::make_unique(*internal_listener_registry); } else { if (config_.has_internal_listener()) { throw EnvoyException(fmt::format("error adding listener '{}': address is not an internal " diff --git a/source/server/listener_impl.h b/source/server/listener_impl.h index cd858d6aaeebe..8e2eee26d7bfe 100644 --- a/source/server/listener_impl.h +++ b/source/server/listener_impl.h @@ -377,11 +377,17 @@ class ListenerImpl final : public Network::ListenerConfig, Network::UdpListenerWorkerRouterPtr listener_worker_router_; }; - struct InternalListenerConfigImpl : public Network::InternalListenerConfig { - InternalListenerConfigImpl( - const envoy::config::listener::v3::Listener_InternalListenerConfig config) - : config_(config) {} - const envoy::config::listener::v3::Listener_InternalListenerConfig config_; + class InternalListenerConfigImpl : public Network::InternalListenerConfig { + public: + InternalListenerConfigImpl(Network::InternalListenerRegistry& internal_listener_registry) + : internal_listener_registry_(internal_listener_registry) {} + + Network::InternalListenerRegistry& internalListenerRegistry() override { + return internal_listener_registry_; + } + + private: + Network::InternalListenerRegistry& internal_listener_registry_; }; /** diff --git a/test/common/event/BUILD b/test/common/event/BUILD index c42e8e1aff1ea..8b945fe028615 100644 --- a/test/common/event/BUILD +++ b/test/common/event/BUILD @@ -16,6 +16,7 @@ envoy_cc_test( "//source/common/event:deferred_task", "//source/common/event:dispatcher_includes", "//source/common/event:dispatcher_lib", + "//source/common/network:address_lib", "//source/common/stats:isolated_store_lib", "//test/mocks:common_lib", "//test/mocks/server:watch_dog_mocks", diff --git a/test/common/event/dispatcher_impl_test.cc b/test/common/event/dispatcher_impl_test.cc index a7c84bb3a5888..75826e6166a6a 100644 --- a/test/common/event/dispatcher_impl_test.cc +++ b/test/common/event/dispatcher_impl_test.cc @@ -11,12 +11,14 @@ #include "source/common/event/deferred_task.h" #include "source/common/event/dispatcher_impl.h" #include "source/common/event/timer_impl.h" +#include "source/common/network/address_impl.h" #include "source/common/stats/isolated_store_impl.h" #include "test/mocks/common.h" #include "test/mocks/event/mocks.h" #include "test/mocks/server/watch_dog.h" #include "test/mocks/stats/mocks.h" +#include "test/test_common/network_utility.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -1549,6 +1551,35 @@ TEST_F(DispatcherWithWatchdogTest, TouchBeforeFdEvent) { dispatcher_->run(Dispatcher::RunType::NonBlock); } +class DispatcherConnectionTest : public testing::Test { +protected: + DispatcherConnectionTest() + : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")) {} + + Api::ApiPtr api_; + DispatcherPtr dispatcher_; +}; + +TEST_F(DispatcherConnectionTest, CreateTcpConnection) { + + auto client_conn = dispatcher_->createClientConnection( + std::make_shared("127.0.0.1", 10911), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + EXPECT_NE(nullptr, client_conn); + client_conn->close(Network::ConnectionCloseType::NoFlush); +} + +// If the internal connection factory is not linked, envoy will be dead when creating connection to +// internal address. +TEST_F(DispatcherConnectionTest, CreateEnvoyInternalConnectionWhenFactoryNotExist) { + EXPECT_DEATH( + dispatcher_->createClientConnection( + std::make_shared("listener_internal_address"), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), + nullptr), + ""); +} + } // namespace } // namespace Event } // namespace Envoy diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index d362401c5881a..245315053d143 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -3039,6 +3039,9 @@ class InternalClientConnectionImplTest : public testing::Test { StrictMock client_callbacks_; }; +// The internal address is passed to Envoy by EDS. If this Envoy instance is configured as internal +// address disabled, the EDS subscription should reject the config before dispatcher attempt to +// establish connection to such address. TEST_F(InternalClientConnectionImplTest, CannotCreateConnectionToInternalAddressWithInternalAddressEnabled) { auto runtime = std::make_unique(); @@ -3048,14 +3051,14 @@ TEST_F(InternalClientConnectionImplTest, "envoy.extensions.network.socket_interface.default_socket_interface"); Network::Address::InstanceConstSharedPtr address = std::make_shared("listener_0", sock_interface); - // Not implemented yet. + ASSERT_DEATH( { ClientConnectionPtr connection = dispatcher_->createClientConnection(address, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); }, - "panic: not implemented"); + ""); } class ClientConnectionWithCustomRawBufferSocketTest : public ConnectionImplTest { diff --git a/test/extensions/bootstrap/internal_listener/BUILD b/test/extensions/bootstrap/internal_listener/BUILD new file mode 100644 index 0000000000000..6df54805e7c81 --- /dev/null +++ b/test/extensions/bootstrap/internal_listener/BUILD @@ -0,0 +1,65 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_extension_cc_test( + name = "client_connection_factory_test", + srcs = ["client_connection_factory_test.cc"], + extension_names = ["envoy.bootstrap.internal_listener"], + deps = [ + "//source/common/common:utility_lib", + "//source/common/network:address_lib", + "//source/extensions/bootstrap/internal_listener:client_connection_factory", + "//source/extensions/io_socket/user_space:io_handle_impl_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/thread_local:thread_local_mocks", + ], +) + +envoy_extension_cc_test( + name = "internal_listener_registry_test", + srcs = ["internal_listener_registry_test.cc"], + extension_names = ["envoy.bootstrap.internal_listener"], + deps = [ + "//source/extensions/bootstrap/internal_listener:config", + "//test/mocks/thread_local:thread_local_mocks", + ], +) + +envoy_extension_cc_test( + name = "internal_listener_integration_test", + srcs = [ + "internal_listener_integration_test.cc", + ], + extension_names = ["envoy.bootstrap.internal_listener"], + deps = [ + "//source/common/config:api_version_lib", + "//source/common/event:dispatcher_includes", + "//source/common/event:dispatcher_lib", + "//source/common/network:connection_lib", + "//source/common/network:utility_lib", + "//source/extensions/bootstrap/internal_listener:config", + "//source/extensions/filters/network/tcp_proxy:config", + "//source/extensions/io_socket/user_space:config", + "//test/common/grpc:grpc_client_integration_lib", + "//test/config:v2_link_hacks", + "//test/integration:http_integration_lib", + "//test/integration/filters:address_restore_listener_filter_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:resources_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/bootstrap/internal_listener/client_connection_factory_test.cc b/test/extensions/bootstrap/internal_listener/client_connection_factory_test.cc new file mode 100644 index 0000000000000..686e6b43f50e8 --- /dev/null +++ b/test/extensions/bootstrap/internal_listener/client_connection_factory_test.cc @@ -0,0 +1,173 @@ +#include "envoy/buffer/buffer.h" +#include "envoy/event/file_event.h" +#include "envoy/network/listen_socket.h" + +#include "source/common/buffer/buffer_impl.h" +#include "source/common/common/fancy_logger.h" +#include "source/common/network/address_impl.h" +#include "source/extensions/bootstrap/internal_listener/client_connection_factory.h" +#include "source/extensions/io_socket/user_space/io_handle_impl.h" + +#include "test/mocks/event/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/test_common/network_utility.h" + +#include "absl/container/fixed_array.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; + +namespace Envoy { +namespace Extensions { +namespace Bootstrap { +namespace InternalListener { + +namespace { + +class MockInternalListenerManger : public Network::InternalListenerManager { +public: + MOCK_METHOD(Network::InternalListenerOptRef, findByAddress, + (const Network::Address::InstanceConstSharedPtr&)); +}; + +// The internal connection factory is linked in this test suite. This test suite verifies the +// connection can be created. +class ClientConnectionFactoryTest : public testing::Test { +public: + ClientConnectionFactoryTest() + : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")), + buf_(1024) { + std::tie(io_handle_, io_handle_peer_) = + IoSocket::UserSpace::IoHandleFactory::createIoHandlePair(); + EXPECT_CALL(tls_allocator_, allocateSlot()); + } + void setupTlsSlot() { + tls_slot_ = + ThreadLocal::TypedSlot::makeUnique( + tls_allocator_); + tls_slot_->set([r = registry_](Event::Dispatcher&) { return r; }); + // TODO: restore the original value via RAII. + Bootstrap::InternalListener::InternalClientConnectionFactory::registry_tls_slot_ = + tls_slot_.get(); + } + Api::ApiPtr api_; + Event::DispatcherPtr dispatcher_; + MockInternalListenerManger internal_listener_manager_; + std::shared_ptr registry_{std::make_shared()}; + ThreadLocal::MockInstance tls_allocator_; + std::unique_ptr> tls_slot_; + + // Owned by IoHandleImpl. + NiceMock* schedulable_cb_; + std::unique_ptr io_handle_; + std::unique_ptr io_handle_peer_; + absl::FixedArray buf_; + Network::Address::EnvoyInternalInstance listener_addr{"listener_internal_address"}; +}; + +class MockInternalListener : public Network::InternalListener { +public: + MOCK_METHOD(void, onAccept, (Network::ConnectionSocketPtr &&)); + MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); +}; + +TEST_F(ClientConnectionFactoryTest, + ConnectFailsIfInternalConnectionThreadLocalRegistryIsNotPublished) { + tls_slot_ = + ThreadLocal::TypedSlot::makeUnique( + tls_allocator_); + // This slot set publish a nullptr. + tls_slot_->set([](Event::Dispatcher&) { return nullptr; }); + Bootstrap::InternalListener::InternalClientConnectionFactory::registry_tls_slot_ = + tls_slot_.get(); + auto client_conn = dispatcher_->createClientConnection( + std::make_shared(listener_addr), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + EXPECT_NE(nullptr, client_conn); + EXPECT_TRUE(client_conn->connecting()); + client_conn->connect(); + // Connect returns error immediately because TLS internal listener registry is not ready. + EXPECT_FALSE(client_conn->connecting()); + client_conn->close(Network::ConnectionCloseType::NoFlush); +} + +TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalConnectionManagerNotExist) { + setupTlsSlot(); + auto client_conn = dispatcher_->createClientConnection( + std::make_shared(listener_addr), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + EXPECT_NE(nullptr, client_conn); + EXPECT_TRUE(client_conn->connecting()); + client_conn->connect(); + // Connect returns error immediately because no internal listener manager is registered. + EXPECT_FALSE(client_conn->connecting()); + client_conn->close(Network::ConnectionCloseType::NoFlush); +} + +TEST_F(ClientConnectionFactoryTest, ConnectFailsIfInternalListenerNotExist) { + setupTlsSlot(); + registry_->setInternalListenerManager(internal_listener_manager_); + + EXPECT_CALL(internal_listener_manager_, findByAddress(_)) + .WillOnce(testing::Return(Network::InternalListenerOptRef())); + + auto client_conn = dispatcher_->createClientConnection( + std::make_shared(listener_addr), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + + EXPECT_NE(nullptr, client_conn); + EXPECT_TRUE(client_conn->connecting()); + client_conn->connect(); + // Connect returns error immediately because no internal listener is ready. + EXPECT_FALSE(client_conn->connecting()); + client_conn->close(Network::ConnectionCloseType::NoFlush); +} + +// Verify that the client connection to envoy internal address can be established. This test case +// does not instantiate a server connection. The server connection is tested in internal listener. +TEST_F(ClientConnectionFactoryTest, ConnectSucceeds) { + setupTlsSlot(); + registry_->setInternalListenerManager(internal_listener_manager_); + MockInternalListener internal_listener; + Network::InternalListenerOptRef internal_listener_opt{internal_listener}; + + EXPECT_CALL(internal_listener_manager_, findByAddress(_)) + .WillOnce(testing::Return(internal_listener_opt)); + Network::ConnectionSocketPtr server_socket; + EXPECT_CALL(internal_listener, onAccept(_)).WillOnce([&](auto&& socket) { + server_socket = std::move(socket); + }); + + auto client_conn = dispatcher_->createClientConnection( + std::make_shared(listener_addr), + Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); + + EXPECT_NE(nullptr, server_socket); + + EXPECT_NE(nullptr, client_conn); + EXPECT_TRUE(client_conn->connecting()); + client_conn->connect(); + + // Connect is successful but the connecting state takes another poll cycle to clear. + EXPECT_TRUE(client_conn->connecting()); + + Buffer::OwnedImpl buf_to_write("0123456789"); + + client_conn->write(buf_to_write, false); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + // The write callback detects that connecting is completed. + EXPECT_FALSE(client_conn->connecting()); + + auto result = server_socket->ioHandle().recv(buf_.data(), buf_.size(), 0); + ASSERT_EQ(10, result.return_value_); + ASSERT_EQ("0123456789", absl::string_view(buf_.data(), result.return_value_)); + + client_conn->close(Network::ConnectionCloseType::NoFlush); + server_socket->close(); +} +} // namespace +} // namespace InternalListener +} // namespace Bootstrap +} // namespace Extensions +} // namespace Envoy diff --git a/test/integration/internal_listener_integration_test.cc b/test/extensions/bootstrap/internal_listener/internal_listener_integration_test.cc similarity index 95% rename from test/integration/internal_listener_integration_test.cc rename to test/extensions/bootstrap/internal_listener/internal_listener_integration_test.cc index 11614b84874a9..20ad6cdffa59b 100644 --- a/test/integration/internal_listener_integration_test.cc +++ b/test/extensions/bootstrap/internal_listener/internal_listener_integration_test.cc @@ -32,6 +32,11 @@ class InternalListenerIntegrationTest : public testing::TestWithParammutable_envoy_internal_address()->set_server_listener_name( "internal_listener"); }); + config_helper_.addBootstrapExtension(R"EOF( +name: envoy.bootstrap.internal_listener +typed_config: + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" +)EOF"); BaseIntegrationTest::initialize(); } }; diff --git a/test/extensions/bootstrap/internal_listener/internal_listener_registry_test.cc b/test/extensions/bootstrap/internal_listener/internal_listener_registry_test.cc new file mode 100644 index 0000000000000..11af2f3946ac6 --- /dev/null +++ b/test/extensions/bootstrap/internal_listener/internal_listener_registry_test.cc @@ -0,0 +1,23 @@ +#include "source/extensions/bootstrap/internal_listener/internal_listener_registry.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Bootstrap { +namespace InternalListener { + +namespace { + +// The tls slot is not initialized yet. +TEST(TlsInternalListenerRegistryTest, TlsSlotNotInitialized) { + TlsInternalListenerRegistry registry; + ASSERT_EQ(nullptr, registry.getLocalRegistry()); +} + +} // namespace +} // namespace InternalListener +} // namespace Bootstrap +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/network/dns_resolver/cares/dns_impl_test.cc b/test/extensions/network/dns_resolver/cares/dns_impl_test.cc index 3728733313caf..4313febe8d466 100644 --- a/test/extensions/network/dns_resolver/cares/dns_impl_test.cc +++ b/test/extensions/network/dns_resolver/cares/dns_impl_test.cc @@ -597,6 +597,8 @@ class CustomInstance : public Address::Instance { } const sockaddr* sockAddr() const override { return instance_.sockAddr(); } socklen_t sockAddrLen() const override { return instance_.sockAddrLen(); } + absl::string_view addressType() const override { PANIC("not implemented"); } + Address::Type type() const override { return instance_.type(); } const SocketInterface& socketInterface() const override { return SocketInterfaceSingleton::get(); diff --git a/test/integration/BUILD b/test/integration/BUILD index d6f93d72335f5..f2e8bb4d480bd 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1676,32 +1676,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "internal_listener_integration_test", - srcs = [ - "internal_listener_integration_test.cc", - ], - deps = [ - ":http_integration_lib", - "//source/common/config:api_version_lib", - "//source/common/event:dispatcher_includes", - "//source/common/event:dispatcher_lib", - "//source/common/network:connection_lib", - "//source/common/network:utility_lib", - "//source/extensions/filters/network/tcp_proxy:config", - "//test/common/grpc:grpc_client_integration_lib", - "//test/config:v2_link_hacks", - "//test/integration/filters:address_restore_listener_filter_lib", - "//test/test_common:network_utility_lib", - "//test/test_common:resources_lib", - "//test/test_common:utility_lib", - "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/network/tcp_proxy/v3:pkg_cc_proto", - "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "cx_limit_integration_test", srcs = ["cx_limit_integration_test.cc"], diff --git a/test/integration/socket_interface_integration_test.cc b/test/integration/socket_interface_integration_test.cc index ddadc7b5d5ab5..dc6d0b186bb0d 100644 --- a/test/integration/socket_interface_integration_test.cc +++ b/test/integration/socket_interface_integration_test.cc @@ -89,7 +89,8 @@ TEST_P(SocketInterfaceIntegrationTest, AddressWithSocketInterface) { client_->close(Network::ConnectionCloseType::FlushWrite); } -// Test that connecting to internal address will crash. +// Test that connecting to internal address will crash if the user space socket extension is not +// linked. TEST_P(SocketInterfaceIntegrationTest, InternalAddressWithSocketInterface) { BaseIntegrationTest::initialize(); @@ -103,7 +104,7 @@ TEST_P(SocketInterfaceIntegrationTest, InternalAddressWithSocketInterface) { ASSERT_DEATH(client_ = dispatcher_->createClientConnection( address, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr), - "panic: not implemented"); + "" /* Nullptr dereference */); } // Test that recv from internal address will crash. diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 2360c129f7f4a..848ee18a78303 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -527,6 +527,7 @@ class MockResolvedAddress : public Address::Instance { MOCK_METHOD(Address::Type, type, (), (const)); MOCK_METHOD(const sockaddr*, sockAddr, (), (const)); MOCK_METHOD(socklen_t, sockAddrLen, (), (const)); + MOCK_METHOD(absl::string_view, addressType, (), (const)); const std::string& asString() const override { return physical_; } absl::string_view asStringView() const override { return physical_; } diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index fe84ce5060816..f8a5ebee2513e 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -111,7 +111,38 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggableinternal_listener_config_ = - std::make_unique(); + std::make_unique(mock_listener_registry_); return listeners_.back().get(); } @@ -335,6 +366,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable dispatcher_{"test"}; + TestListener::MockInternalListenerRegistery mock_listener_registry_; std::list listeners_; std::unique_ptr handler_; NiceMock manager_; diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index f996c74889d60..f228ed1898281 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -48,6 +48,9 @@ using testing::Return; using testing::ReturnRef; using testing::Throw; +// For internal listener test only. +SINGLETON_MANAGER_REGISTRATION(internal_listener_registry); + class ListenerManagerImplWithDispatcherStatsTest : public ListenerManagerImplTest { protected: ListenerManagerImplWithDispatcherStatsTest() { enable_dispatcher_stats_ = true; } @@ -55,6 +58,13 @@ class ListenerManagerImplWithDispatcherStatsTest : public ListenerManagerImplTes class ListenerManagerImplWithRealFiltersTest : public ListenerManagerImplTest { public: + void SetUp() override { + ListenerManagerImplTest::SetUp(); + ASSERT_NE(nullptr, server_.singletonManager().getTyped( + "internal_listener_registry_singleton", + [registry = internal_registry_]() { return registry; })); + } + /** * Create an IPv4 listener with a given name. */ diff --git a/test/server/listener_manager_impl_test.h b/test/server/listener_manager_impl_test.h index 6febd935efa37..42487dc765981 100644 --- a/test/server/listener_manager_impl_test.h +++ b/test/server/listener_manager_impl_test.h @@ -306,6 +306,14 @@ class ListenerManagerImplTest : public testing::Test { TestThreadsafeSingletonInjector os_calls_{&os_sys_calls_}; Api::OsSysCallsImpl os_sys_calls_actual_; NiceMock server_; + class DumbInternalListenerRegistry : public Singleton::Instance, + public Network::InternalListenerRegistry { + public: + MOCK_METHOD(Network::LocalInternalListenerRegistry*, getLocalRegistry, ()); + }; + std::shared_ptr internal_registry_{ + std::make_shared()}; + NiceMock listener_factory_; NiceMock validation_visitor; MockWorker* worker_ = new MockWorker();