connection: Add TCP_FASTOPEN listener option

bazel/repository_locations.bzl

@@ -76,8 +76,8 @@ REPOSITORY_LOCATIONS = dict(
         urls = ["https://github.com/google/protobuf/archive/v3.5.0.tar.gz"],
     ),
     envoy_api = dict(
-        commit = "92ffa417b5d2e8ab3912ff4da4daa6aa93f8df0f",
-        remote = "https://github.com/envoyproxy/data-plane-api",
+        commit = "28c23f754539369c57b28359067f100059ce0d0b",
+        remote = "https://github.com/bmetzdorf/data-plane-api.git", # TODO FIXME
     ),
     grpc_httpjson_transcoding = dict(
         commit = "e4f58aa07b9002befa493a0a82e10f2e98b51fc6",

include/envoy/network/listen_socket.h

@@ -31,7 +31,7 @@ class Socket {
    */
   virtual void close() PURE;
 
-  enum class SocketState { PreBind, PostBind };
+  enum class SocketState { PreBind, PostBind, Listening };
 
   /**
    * Visitor class for setting socket options.

source/common/network/listener_impl.cc

@@ -1,5 +1,6 @@
 #include "common/network/listener_impl.h"
 
+#include <netinet/tcp.h>
 #include <sys/un.h>
 
 #include "envoy/common/exception.h"
@@ -66,6 +67,14 @@ ListenerImpl::ListenerImpl(Event::DispatcherImpl& dispatcher, Socket& socket, Li
           fmt::format("cannot listen on socket: {}", socket.localAddress()->asString()));
     }
 
+    for (auto& option : *socket.options()) {
+      if (!option->setOption(socket, Socket::SocketState::Listening)) {
+        throw CreateListenerException(
+            fmt::format("cannot set post-listen socket option on socket: {}",
+                        socket.localAddress()->asString()));
+      }
+    }
+
     evconnlistener_set_error_cb(listener_.get(), errorCallback);
   }
 }

source/common/network/socket_option_impl.cc

@@ -32,8 +32,25 @@ bool SocketOptionImpl::setOption(Socket& socket, Socket::SocketState state) cons
         return false;
       }
     }
-  }
+  } else if (state == Socket::SocketState::Listening) {
+    if (accept_tcp_fast_open_.has_value()) {
+      const int tfo_value = accept_tcp_fast_open_.value() ? ENVOY_TCP_FASTOPEN_BACKLOG : 0;
+      const SocketOptionName option_name = ENVOY_SOCKET_TCP_FASTOPEN;
+      int error = -1;
+      if (option_name) {
+        error = Api::OsSysCallsSingleton::get().setsockopt(
+            socket.fd(), IPPROTO_TCP, option_name.value(),
+            reinterpret_cast<const void*>(&tfo_value), sizeof(tfo_value));
+      } else {
+        error = ENOTSUP;
+      }
 
+      if (error != 0) {
+        ENVOY_LOG(warn, "Setting IP_TCP_FASTOPEN on listener socket failed: {}", strerror(error));
+        return false;
+      }
+    }
+  }
   return true;
 }
 

source/common/network/socket_option_impl.h

@@ -41,10 +41,28 @@ typedef absl::optional<int> SocketOptionName;
 #define ENVOY_SOCKET_IPV6_FREEBIND Network::SocketOptionName()
 #endif
 
+#ifdef TCP_FASTOPEN
+#define ENVOY_SOCKET_TCP_FASTOPEN Network::SocketOptionName(TCP_FASTOPEN)
+#else
+#define ENVOY_SOCKET_TCP_FASTOPEN Network::SocketOptionName()
+#endif
+
+// On macOS the socket MUST be listening already for TCP_FASTOPEN to be set and backlog MUST be 1
+// (the actual value is set via the net.inet.tcp.fastopen_backlog kernel parameter.
+// For Linux we default to 128, which libevent is using in
+// https://github.com/libevent/libevent/blob/release-2.1.8-stable/listener.c#L176
+#if defined(__APPLE__)
+#define ENVOY_TCP_FASTOPEN_BACKLOG 1
+#else
+#define ENVOY_TCP_FASTOPEN_BACKLOG 128
+#endif
+
 class SocketOptionImpl : public Socket::Option, Logger::Loggable<Logger::Id::connection> {
 public:
-  SocketOptionImpl(absl::optional<bool> transparent, absl::optional<bool> freebind)
-      : transparent_(transparent), freebind_(freebind) {}
+  SocketOptionImpl(absl::optional<bool> transparent, absl::optional<bool> freebind,
+                   absl::optional<bool> accept_tcp_fast_open)
+      : transparent_(transparent), freebind_(freebind),
+        accept_tcp_fast_open_(accept_tcp_fast_open) {}
 
   // Socket::Option
   bool setOption(Socket& socket, Socket::SocketState state) const override;
@@ -72,6 +90,7 @@ class SocketOptionImpl : public Socket::Option, Logger::Loggable<Logger::Id::con
 private:
   const absl::optional<bool> transparent_;
   const absl::optional<bool> freebind_;
+  const absl::optional<bool> accept_tcp_fast_open_;
 };
 
 } // namespace Network

source/common/upstream/upstream_impl.cc

@@ -75,9 +75,11 @@ uint64_t parseFeatures(const envoy::api::v2::Cluster& config,
 class UpstreamSocketOption : public Network::SocketOptionImpl {
 public:
   UpstreamSocketOption(const ClusterInfo& cluster_info)
-      : Network::SocketOptionImpl({}, cluster_info.features() & ClusterInfo::Features::FREEBIND
-                                          ? true
-                                          : absl::optional<bool>{}) {}
+      : Network::SocketOptionImpl({},
+                                  cluster_info.features() & ClusterInfo::Features::FREEBIND
+                                      ? true
+                                      : absl::optional<bool>{},
+                                  {}) {}
 };
 
 } // namespace

source/server/listener_manager_impl.cc

@@ -114,7 +114,9 @@ class ListenerSocketOption : public Network::SocketOptionImpl {
   ListenerSocketOption(const envoy::api::v2::Listener& config)
       : Network::SocketOptionImpl(
             PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, transparent, absl::optional<bool>{}),
-            PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, freebind, absl::optional<bool>{})) {}
+            PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, freebind, absl::optional<bool>{}),
+            PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, accept_tcp_fast_open, absl::optional<bool>{})) {
+  }
 };
 
 ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, ListenerManagerImpl& parent,

test/common/network/socket_option_impl_test.cc

@@ -23,6 +23,24 @@ class SocketOptionImplTest : public testing::Test {
   NiceMock<MockListenSocket> socket_;
   Api::MockOsSysCalls os_sys_calls_;
   TestThreadsafeSingletonInjector<Api::OsSysCallsImpl> os_calls{&os_sys_calls_};
+
+  void testSetSocketOptionSuccess(SocketOptionImpl& socket_option, int socket_level,
+                                  Network::SocketOptionName option_name, int option_val,
+                                  Socket::SocketState when) {
+    if (option_name.has_value()) {
+      Address::Ipv4Instance address("1.2.3.4", 5678);
+      const int fd = address.socket(Address::SocketType::Stream);
+      EXPECT_CALL(socket_, fd()).WillRepeatedly(Return(fd));
+      EXPECT_CALL(os_sys_calls_, setsockopt_(_, socket_level, option_name.value(), _, sizeof(int)))
+          .WillOnce(Invoke([option_val](int, int, int, const void* optval, socklen_t) -> int {
+            EXPECT_EQ(option_val, *static_cast<const int*>(optval));
+            return 0;
+          }));
+      EXPECT_TRUE(socket_option.setOption(socket_, when));
+    } else {
+      EXPECT_FALSE(socket_option.setOption(socket_, when));
+    }
+  }
 };
 
 // We fail to set the option if the socket FD is bad.
@@ -33,105 +51,63 @@ TEST_F(SocketOptionImplTest, BadFd) {
 
 // Nop when there are no socket options set.
 TEST_F(SocketOptionImplTest, SetOptionEmptyNop) {
-  SocketOptionImpl socket_option{{}, {}};
+  SocketOptionImpl socket_option{{}, {}, {}};
   EXPECT_TRUE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
   EXPECT_TRUE(socket_option.setOption(socket_, Socket::SocketState::PostBind));
+  EXPECT_TRUE(socket_option.setOption(socket_, Socket::SocketState::Listening));
 }
 
 // We fail to set the option when the underlying setsockopt syscall fails.
 TEST_F(SocketOptionImplTest, SetOptionTransparentFailure) {
-  SocketOptionImpl socket_option{true, {}};
+  SocketOptionImpl socket_option{true, {}, {}};
   EXPECT_FALSE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
 }
 
 // We fail to set the option when the underlying setsockopt syscall fails.
 TEST_F(SocketOptionImplTest, SetOptionFreebindFailure) {
-  SocketOptionImpl socket_option{{}, true};
+  SocketOptionImpl socket_option{{}, true, {}};
   EXPECT_FALSE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
 }
 
+// We fail to set the tcp-fastopen option when the underlying setsockopt syscall fails.
+TEST_F(SocketOptionImplTest, SetOptionTcpFastopenFailure) {
+  SocketOptionImpl socket_option{{}, {}, true};
+  EXPECT_FALSE(socket_option.setOption(socket_, Socket::SocketState::Listening));
+}
+
 // The happy path for setOption(); IP_TRANSPARENT is set to true.
 TEST_F(SocketOptionImplTest, SetOptionTransparentSuccessTrue) {
-  SocketOptionImpl socket_option{true, {}};
-  if (ENVOY_SOCKET_IP_TRANSPARENT.has_value()) {
-    Address::Ipv4Instance address("1.2.3.4", 5678);
-    const int fd = address.socket(Address::SocketType::Stream);
-    EXPECT_CALL(socket_, fd()).WillRepeatedly(Return(fd));
-    EXPECT_CALL(os_sys_calls_,
-                setsockopt_(_, IPPROTO_IP, ENVOY_SOCKET_IP_TRANSPARENT.value(), _, sizeof(int)))
-        .WillOnce(Invoke([](int, int, int, const void* optval, socklen_t) -> int {
-          EXPECT_EQ(1, *static_cast<const int*>(optval));
-          return 0;
-        }));
-    EXPECT_TRUE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
-  } else {
-    EXPECT_FALSE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
-  }
+  SocketOptionImpl socket_option{true, {}, {}};
+  testSetSocketOptionSuccess(socket_option, IPPROTO_IP, ENVOY_SOCKET_IP_TRANSPARENT, 1,
+                             Socket::SocketState::PreBind);
 }
 
 // The happy path for setOption(); IP_FREEBIND is set to true.
 TEST_F(SocketOptionImplTest, SetOptionFreebindSuccessTrue) {
-  SocketOptionImpl socket_option{{}, true};
-  if (ENVOY_SOCKET_IP_FREEBIND.has_value()) {
-    Address::Ipv4Instance address("1.2.3.4", 5678);
-    const int fd = address.socket(Address::SocketType::Stream);
-    EXPECT_CALL(socket_, fd()).WillRepeatedly(Return(fd));
-    EXPECT_CALL(os_sys_calls_,
-                setsockopt_(_, IPPROTO_IP, ENVOY_SOCKET_IP_FREEBIND.value(), _, sizeof(int)))
-        .WillOnce(Invoke([](int, int, int, const void* optval, socklen_t) -> int {
-          EXPECT_EQ(1, *static_cast<const int*>(optval));
-          return 0;
-        }));
-    EXPECT_TRUE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
-  } else {
-    EXPECT_FALSE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
-  }
+  SocketOptionImpl socket_option{{}, true, {}};
+  testSetSocketOptionSuccess(socket_option, IPPROTO_IP, ENVOY_SOCKET_IP_FREEBIND, 1,
+                             Socket::SocketState::PreBind);
 }
 
 // The happy path for setOpion(); IP_TRANSPARENT is set to false.
 TEST_F(SocketOptionImplTest, SetOptionTransparentSuccessFalse) {
-  SocketOptionImpl socket_option{false, {}};
-  if (ENVOY_SOCKET_IP_TRANSPARENT.has_value()) {
-    Address::Ipv4Instance address("1.2.3.4", 5678);
-    const int fd = address.socket(Address::SocketType::Stream);
-    EXPECT_CALL(socket_, fd()).WillRepeatedly(Return(fd));
-    EXPECT_CALL(os_sys_calls_,
-                setsockopt_(_, IPPROTO_IP, ENVOY_SOCKET_IP_TRANSPARENT.value(), _, sizeof(int)))
-        .Times(2)
-        .WillRepeatedly(Invoke([](int, int, int, const void* optval, socklen_t) -> int {
-          EXPECT_EQ(0, *static_cast<const int*>(optval));
-          return 0;
-        }));
-    EXPECT_TRUE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
-    EXPECT_TRUE(socket_option.setOption(socket_, Socket::SocketState::PostBind));
-  } else {
-    EXPECT_FALSE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
-    EXPECT_FALSE(socket_option.setOption(socket_, Socket::SocketState::PostBind));
-  }
+  SocketOptionImpl socket_option{false, {}, {}};
+  testSetSocketOptionSuccess(socket_option, IPPROTO_IP, ENVOY_SOCKET_IP_TRANSPARENT, 0,
+                             Socket::SocketState::PreBind);
+  testSetSocketOptionSuccess(socket_option, IPPROTO_IP, ENVOY_SOCKET_IP_TRANSPARENT, 0,
+                             Socket::SocketState::PostBind);
 }
 
 // The happy path for setOpion(); IP_FREEBIND is set to false.
 TEST_F(SocketOptionImplTest, SetOptionFreebindSuccessFalse) {
-  SocketOptionImpl socket_option{{}, false};
-  if (ENVOY_SOCKET_IP_FREEBIND.has_value()) {
-    Address::Ipv4Instance address("1.2.3.4", 5678);
-    const int fd = address.socket(Address::SocketType::Stream);
-    EXPECT_CALL(socket_, fd()).WillRepeatedly(Return(fd));
-    EXPECT_CALL(os_sys_calls_,
-                setsockopt_(_, IPPROTO_IP, ENVOY_SOCKET_IP_FREEBIND.value(), _, sizeof(int)))
-        .WillOnce(Invoke([](int, int, int, const void* optval, socklen_t) -> int {
-          EXPECT_EQ(0, *static_cast<const int*>(optval));
-          return 0;
-        }));
-    EXPECT_TRUE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
-  } else {
-    EXPECT_FALSE(socket_option.setOption(socket_, Socket::SocketState::PreBind));
-  }
+  SocketOptionImpl socket_option{{}, false, {}};
+  testSetSocketOptionSuccess(socket_option, IPPROTO_IP, ENVOY_SOCKET_IP_FREEBIND, 0,
+                             Socket::SocketState::PreBind);
 }
 
 // Freebind settings have no effect on post-bind behavior.
 TEST_F(SocketOptionImplTest, SetOptionFreebindPostBind) {
-  SocketOptionImpl socket_option{{}, true};
+  SocketOptionImpl socket_option{{}, true, {}};
   EXPECT_TRUE(socket_option.setOption(socket_, Socket::SocketState::PostBind));
 }