connection: Add TCP_FASTOPEN listener option

docs/root/intro/version_history.rst

@@ -24,6 +24,7 @@ Version history
   <envoy_api_field_core.HealthCheck.unhealthy_edge_interval>`, :ref:`unhealthy to healthy
   <envoy_api_field_core.HealthCheck.healthy_edge_interval>` and for subsequent checks on
   :ref:`unhealthy hosts <envoy_api_field_core.HealthCheck.unhealthy_interval>`.
+* listeners: added :ref:`tcp_fast_open_queue_length <envoy_api_field_Listener.tcp_fast_open_queue_length>` option.
 * health check: added support for :ref:`custom health check <envoy_api_field_core.HealthCheck.custom_health_check>`.
 * load balancing: added :ref:`weighted round robin
   <arch_overview_load_balancing_types_round_robin>` support. The round robin

include/envoy/network/listen_socket.h

@@ -31,7 +31,14 @@ class Socket {
    */
   virtual void close() PURE;
 
-  enum class SocketState { PreBind, PostBind };
+  enum class SocketState {
+    // Socket options are applied after socket creation but before binding the socket to a port
+    PreBind,
+    // Socket options are applied after binding the socket to a port but before calling listen()
+    PostBind,
+    // Socket options are applied after calling listen()
+    Listening,
+  };
 
   /**
    * Visitor class for setting socket options.

source/common/network/listener_impl.cc

@@ -66,6 +66,17 @@ ListenerImpl::ListenerImpl(Event::DispatcherImpl& dispatcher, Socket& socket, Li
           fmt::format("cannot listen on socket: {}", socket.localAddress()->asString()));
     }
 
+    Socket::OptionsSharedPtr options = socket.options();
+    if (options != nullptr) {
+      for (auto& option : *options) {
+        if (!option->setOption(socket, Socket::SocketState::Listening)) {
+          throw CreateListenerException(
+              fmt::format("cannot set post-listen socket option on socket: {}",
+                          socket.localAddress()->asString()));
+        }
+      }
+    }
+
     evconnlistener_set_error_cb(listener_.get(), errorCallback);
   }
 }

source/common/network/socket_option_impl.cc

@@ -10,14 +10,16 @@ namespace Envoy {
 namespace Network {
 
 bool SocketOptionImpl::setOption(Socket& socket, Socket::SocketState state) const {
-  if (transparent_.has_value()) {
-    const int should_transparent = transparent_.value() ? 1 : 0;
-    const int error =
-        setIpSocketOption(socket, ENVOY_SOCKET_IP_TRANSPARENT, ENVOY_SOCKET_IPV6_TRANSPARENT,
-                          &should_transparent, sizeof(should_transparent));
-    if (error != 0) {
-      ENVOY_LOG(warn, "Setting IP_TRANSPARENT on listener socket failed: {}", strerror(errno));
-      return false;
+  if (state == Socket::SocketState::PreBind || state == Socket::SocketState::PostBind) {
+    if (transparent_.has_value()) {
+      const int should_transparent = transparent_.value() ? 1 : 0;
+      const int error =
+          setIpSocketOption(socket, ENVOY_SOCKET_IP_TRANSPARENT, ENVOY_SOCKET_IPV6_TRANSPARENT,
+                            &should_transparent, sizeof(should_transparent));
+      if (error != 0) {
+        ENVOY_LOG(warn, "Setting IP_TRANSPARENT on listener socket failed: {}", strerror(error));
+        return false;
+      }
     }
   }
 

source/common/network/socket_option_impl.h

@@ -41,7 +41,13 @@ typedef absl::optional<int> SocketOptionName;
 #define ENVOY_SOCKET_IPV6_FREEBIND Network::SocketOptionName()
 #endif
 
-class SocketOptionImpl : public Socket::Option, Logger::Loggable<Logger::Id::connection> {
+#ifdef TCP_FASTOPEN
+#define ENVOY_SOCKET_TCP_FASTOPEN Network::SocketOptionName(TCP_FASTOPEN)
+#else
+#define ENVOY_SOCKET_TCP_FASTOPEN Network::SocketOptionName()
+#endif
+
+class SocketOptionImpl : public Socket::Option, protected Logger::Loggable<Logger::Id::connection> {
 public:
   SocketOptionImpl(absl::optional<bool> transparent, absl::optional<bool> freebind)
       : transparent_(transparent), freebind_(freebind) {}

source/server/BUILD

@@ -197,6 +197,7 @@ envoy_cc_library(
         ":configuration_lib",
         ":drain_manager_lib",
         ":init_manager_lib",
+        ":listener_socket_option_lib",
         "//include/envoy/server:filter_config_interface",
         "//include/envoy/server:listener_manager_interface",
         "//include/envoy/server:transport_socket_config_interface",
@@ -215,6 +216,19 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "listener_socket_option_lib",
+    srcs = ["listener_socket_option_impl.cc"],
+    hdrs = ["listener_socket_option_impl.h"],
+    deps = [
+        ":configuration_lib",
+        "//source/common/api:os_sys_calls_lib",
+        "//source/common/network:listen_socket_lib",
+        "//source/common/network:socket_option_lib",
+        "//source/common/protobuf:utility_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "proto_descriptors_lib",
     srcs = ["proto_descriptors.cc"],

source/server/listener_manager_impl.cc

@@ -15,6 +15,7 @@
 
 #include "server/configuration_impl.h"
 #include "server/drain_manager_impl.h"
+#include "server/listener_socket_option_impl.h"
 
 #include "extensions/filters/listener/well_known_names.h"
 #include "extensions/transport_sockets/well_known_names.h"
@@ -110,16 +111,6 @@ ProdListenerComponentFactory::createDrainManager(envoy::api::v2::Listener::Drain
   return DrainManagerPtr{new DrainManagerImpl(server_, drain_type)};
 }
 
-// Socket::Option implementation for API-defined listener socket options.
-// This same object can be extended to handle additional listener socket options.
-class ListenerSocketOption : public Network::SocketOptionImpl {
-public:
-  ListenerSocketOption(const envoy::api::v2::Listener& config)
-      : Network::SocketOptionImpl(
-            PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, transparent, absl::optional<bool>{}),
-            PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, freebind, absl::optional<bool>{})) {}
-};
-
 ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, ListenerManagerImpl& parent,
                            const std::string& name, bool modifiable, bool workers_started,
                            uint64_t hash)
@@ -142,7 +133,7 @@ ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, ListenerManag
   ASSERT(config.filter_chains().size() >= 1);
 
   // Add listen socket options from the config.
-  addListenSocketOption(std::make_shared<ListenerSocketOption>(config));
+  addListenSocketOption(std::make_shared<ListenerSocketOptionImpl>(config));
 
   if (!config.listener_filters().empty()) {
     listener_filter_factories_ =
@@ -306,6 +297,10 @@ void ListenerImpl::setSocket(const Network::SocketSharedPtr& socket) {
       } else {
         ENVOY_LOG(debug, "{}", message);
       }
+
+      // Add the options to the socket_ so that SocketState::Listening options can be
+      // set in the worker after listen()/evconnlistener_new() is called.
+      socket_->addOption(option);
     }
   }
 }

source/server/listener_socket_option_impl.cc

@@ -0,0 +1,53 @@
+#include "server/listener_socket_option_impl.h"
+
+#include "common/api/os_sys_calls_impl.h"
+
+namespace Envoy {
+namespace Server {
+
+ListenerSocketOptionImpl::ListenerSocketOptionImpl(const envoy::api::v2::Listener& config)
+    : Network::SocketOptionImpl(
+          PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, transparent, absl::optional<bool>{}),
+          PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, freebind, absl::optional<bool>{})),
+      tcp_fast_open_queue_length_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(
+          config, tcp_fast_open_queue_length, absl::optional<uint32_t>{})) {}
+
+ListenerSocketOptionImpl::ListenerSocketOptionImpl(
+    absl::optional<bool> transparent, absl::optional<bool> freebind,
+    absl::optional<uint32_t> tcp_fast_open_queue_length)
+    : Network::SocketOptionImpl(transparent, freebind),
+      tcp_fast_open_queue_length_(tcp_fast_open_queue_length) {}
+
+bool ListenerSocketOptionImpl::setOption(Network::Socket& socket,
+                                         Network::Socket::SocketState state) const {
+  bool result = Network::SocketOptionImpl::setOption(socket, state);
+  if (!result) {
+    return result;
+  }
+
+  if (state == Network::Socket::SocketState::Listening) {
+    if (tcp_fast_open_queue_length_.has_value()) {
+      const int tfo_value = tcp_fast_open_queue_length_.value();
+      const Network::SocketOptionName option_name = ENVOY_SOCKET_TCP_FASTOPEN;
+      if (option_name) {
+        const int error = Api::OsSysCallsSingleton::get().setsockopt(
+            socket.fd(), IPPROTO_TCP, option_name.value(),
+            reinterpret_cast<const void*>(&tfo_value), sizeof(tfo_value));
+        if (error != 0) {
+          ENVOY_LOG(warn, "Setting TCP_FASTOPEN on listener socket failed: {}", strerror(errno));
+          return false;
+        } else {
+          ENVOY_LOG(debug, "Successfully set socket option TCP_FASTOPEN to {}", tfo_value);
+        }
+      } else {
+        ENVOY_LOG(warn, "Unsupported socket option TCP_FASTOPEN");
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+} // namespace Server
+} // namespace Envoy

source/server/listener_socket_option_impl.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <netinet/tcp.h>
+
+#include "envoy/api/v2/listener/listener.pb.h"
+
+#include "common/network/listen_socket_impl.h"
+#include "common/network/socket_option_impl.h"
+
+#include "server/configuration_impl.h"
+
+namespace Envoy {
+namespace Server {
+
+// Socket::Option implementation for API-defined listener socket options.
+// This same object can be extended to handle additional listener socket options.
+class ListenerSocketOptionImpl : public Network::SocketOptionImpl {
+public:
+  ListenerSocketOptionImpl(const envoy::api::v2::Listener& config);
+
+  ListenerSocketOptionImpl(absl::optional<bool> transparent, absl::optional<bool> freebind,
+                           absl::optional<uint32_t> tcp_fast_open_queue_length);
+
+  // Socket::Option
+  bool setOption(Network::Socket& socket, Network::Socket::SocketState state) const override;
+
+private:
+  const absl::optional<uint32_t> tcp_fast_open_queue_length_;
+};
+
+} // namespace Server
+} // namespace Envoy

test/common/network/BUILD

@@ -3,6 +3,7 @@ licenses(["notice"])  # Apache 2
 load(
     "//bazel:envoy_build_system.bzl",
     "envoy_cc_test",
+    "envoy_cc_test_library",
     "envoy_package",
 )
 
@@ -152,19 +153,28 @@ envoy_cc_test(
     ],
 )
 
-envoy_cc_test(
-    name = "socket_option_impl_test",
-    srcs = ["socket_option_impl_test.cc"],
+envoy_cc_test_library(
+    name = "socket_option_test_harness",
+    srcs = ["socket_option_test_harness.h"],
     deps = [
         "//source/common/network:address_lib",
         "//source/common/network:socket_option_lib",
+        "//source/server:listener_socket_option_lib",
         "//test/mocks/api:api_mocks",
         "//test/mocks/network:network_mocks",
         "//test/test_common:logging_lib",
         "//test/test_common:threadsafe_singleton_injector_lib",
     ],
 )
 
+envoy_cc_test(
+    name = "socket_option_impl_test",
+    srcs = ["socket_option_impl_test.cc"],
+    deps = [
+        ":socket_option_test_harness",
+    ],
+)
+
 envoy_cc_test(
     name = "utility_test",
     srcs = ["utility_test.cc"],

test/common/network/listener_impl_test.cc

@@ -86,6 +86,35 @@ INSTANTIATE_TEST_CASE_P(IpVersions, ListenerImplTest,
                         testing::ValuesIn(TestEnvironment::getIpVersionsForTest()),
                         TestUtility::ipTestParamsToString);
 
+// Test that socket options are set after the listener is setup.
+TEST_P(ListenerImplTest, SetListeningSocketOptionsSuccess) {
+  Event::DispatcherImpl dispatcher;
+  Network::MockListenerCallbacks listener_callbacks;
+  Network::MockConnectionHandler connection_handler;
+
+  Network::TcpListenSocket socket(Network::Test::getCanonicalLoopbackAddress(version_), nullptr,
+                                  true);
+  std::shared_ptr<MockSocketOption> option = std::make_shared<MockSocketOption>();
+  socket.addOption(option);
+  EXPECT_CALL(*option, setOption(_, Socket::SocketState::Listening)).WillOnce(Return(true));
+  TestListenerImpl listener(dispatcher, socket, listener_callbacks, true, false);
+}
+
+// Test that an exception is thrown if there is an error setting socket options.
+TEST_P(ListenerImplTest, SetListeningSocketOptionsError) {
+  Event::DispatcherImpl dispatcher;
+  Network::MockListenerCallbacks listener_callbacks;
+  Network::MockConnectionHandler connection_handler;
+
+  Network::TcpListenSocket socket(Network::Test::getCanonicalLoopbackAddress(version_), nullptr,
+                                  true);
+  std::shared_ptr<MockSocketOption> option = std::make_shared<MockSocketOption>();
+  socket.addOption(option);
+  EXPECT_CALL(*option, setOption(_, Socket::SocketState::Listening)).WillOnce(Return(false));
+  EXPECT_THROW(TestListenerImpl(dispatcher, socket, listener_callbacks, true, false),
+               CreateListenerException);
+}
+
 TEST_P(ListenerImplTest, UseActualDst) {
   Stats::IsolatedStoreImpl stats_store;
   Event::DispatcherImpl dispatcher;