io: add iohandle backed with io_uring

CODEOWNERS

@@ -168,6 +168,8 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp
 /*/extensions/matching/common_inputs/environment @snowp @donyu
 # user space socket pair and event
 /*/extensions/io_socket/user_space @lambdai @antoniovicente
+# io_uring-based IO socket
+/*/extensions/io_socket/io_uring @rojkov @antoniovicente
 # Default UUID4 request ID extension
 /*/extensions/request_id/uuid @mattklein123 @alyssawilk
 # HTTP header formatters

api/envoy/extensions/network/socket_interface/v3/io_uring_socket_interface.proto

@@ -0,0 +1,32 @@
+syntax = "proto3";
+
+package envoy.extensions.network.socket_interface.v3;
+
+import "google/protobuf/wrappers.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.network.socket_interface.v3";
+option java_outer_classname = "IoUringSocketInterfaceProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: ``io_uring`` Socket Interface configuration]
+
+// Configuration for a socket interface that relies on Linux specific ``io_uring`` API to create
+// sockets.
+message IoUringSocketInterface {
+  // The size of read buffer. If not set, defaults to 8192.
+  google.protobuf.UInt32Value read_buffer_size = 1 [(validate.rules).uint32 = {gte: 4096}];
+
+  // The size of ``io_uring`` queues. For heavily loaded processes 300 is a
+  // good enough value. If the load is not high and memory is a constraint
+  // then it's safe to have smaller queues. If not set, defaults to 300.
+  google.protobuf.UInt32Value io_uring_size = 2 [(validate.rules).uint32 = {gte: 16}];
+
+  // When this flag is specified, a kernel thread is created to perform submission queue
+  // polling. An ``io_uring`` instance configured in this way enables ``io_uring`` sockets to issue
+  // I/O without ever context switching into the kernel.
+  bool use_submission_queue_polling = 3;
+}

bazel/foreign_cc/BUILD

@@ -6,6 +6,13 @@ licenses(["notice"])  # Apache 2
 
 envoy_package()
 
+configure_make(
+    name = "liburing",
+    configure_in_place = True,
+    lib_source = "@com_github_axboe_liburing//:all",
+    tags = ["skip_on_windows"],
+)
+
 # autotools packages are unusable on Windows as-is
 # TODO: Consider our own gperftools.BUILD file as we do with many other packages
 configure_make(

bazel/repositories.bzl

@@ -175,6 +175,7 @@ def envoy_dependencies(skip_targets = []):
     # The long repo names (`com_github_fmtlib_fmt` instead of `fmtlib`) are
     # semi-standard in the Bazel community, intended to avoid both duplicate
     # dependencies and name conflicts.
+    _com_github_axboe_liburing()
     _com_github_c_ares_c_ares()
     _com_github_circonus_labs_libcircllhist()
     _com_github_cyan4973_xxhash()
@@ -290,6 +291,16 @@ def _com_github_circonus_labs_libcircllhist():
         actual = "@com_github_circonus_labs_libcircllhist//:libcircllhist",
     )
 
+def _com_github_axboe_liburing():
+    external_http_archive(
+        name = "com_github_axboe_liburing",
+        build_file_content = BUILD_ALL_CONTENT,
+    )
+    native.bind(
+        name = "uring",
+        actual = "@envoy//bazel/foreign_cc:liburing",
+    )
+
 def _com_github_c_ares_c_ares():
     external_http_archive(
         name = "com_github_c_ares_c_ares",

bazel/repository_locations.bzl

@@ -113,6 +113,21 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         release_date = "2021-06-03",
         cpe = "N/A",
     ),
+    com_github_axboe_liburing = dict(
+        project_name = "liburing",
+        project_desc = "C helpers to set up and tear down io_uring instances",
+        project_url = "https://github.com/axboe/liburing",
+        version = "2.1",
+        sha256 = "f1e0500cb3934b0b61c5020c3999a973c9c93b618faff1eba75aadc95bb03e07",
+        strip_prefix = "liburing-liburing-{version}",
+        urls = ["https://github.com/axboe/liburing/archive/liburing-{version}.tar.gz"],
+        use_category = ["dataplane_ext", "controlplane"],
+        extensions = [
+            "envoy.io_socket.io_uring",
+        ],
+        release_date = "2021-09-09",
+        cpe = "N/A",
+    ),
     com_github_c_ares_c_ares = dict(
         project_name = "c-ares",
         project_desc = "C library for asynchronous DNS requests",

docs/root/api-v3/bootstrap/bootstrap.rst

@@ -6,6 +6,7 @@ Bootstrap
   :maxdepth: 2
 
   ../config/bootstrap/v3/bootstrap.proto
+  ../extensions/network/socket_interface/v3/io_uring_socket_interface.proto
   ../config/metrics/v3/stats.proto
   ../config/metrics/v3/metrics_service.proto
   ../config/overload/v3/overload.proto

docs/root/configuration/other_features/io_uring.rst

@@ -0,0 +1,26 @@
+.. _config_sock_interface_io_uring:
+
+io_uring Socket Interface
+=========================
+
+* :ref:`v3 API reference <envoy_v3_api_msg_extensions.network.socket_interface.v3.IoUringSocketInterface>`
+
+.. attention::
+
+  The io_uring socket interface extension is experimental and is currently under active development.
+
+io_uring is an asynchronous I/O API implemented in the Linux kernel.
+This socket interface uses [liburing](https://github.com/axboe/liburing) to integrate io_uring
+with Envoy.
+
+Example configuration
+---------------------
+
+.. code-block:: yaml
+
+  bootstrap_extensions:
+    - name: envoy.extensions.io_socket.io_uring
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.network.socket_interface.v3.IoUringSocketInterface
+  default_socket_interface: "envoy.extensions.network.socket_interface.io_uring"
+

docs/root/configuration/other_features/other_features.rst

@@ -4,6 +4,7 @@ Other features
 .. toctree::
   :maxdepth: 2
 
+  io_uring
   rate_limit
   vcl
   wasm

docs/root/faq/windows/win_not_supported_features.rst

@@ -10,6 +10,7 @@ The most notable features that are not supported on Windows are:
 * :ref:`Hot restart <arch_overview_hot_restart>`
 * :ref:`Signed Exchange Filter <config_http_filters_sxg>`
 * :ref:`VCL Socket Interface <config_sock_interface_vcl>`
+* :ref:`io_uring Socket Interface <config_sock_interface_io_uring>`
 
 There are certain Envoy features that require newer versions of Windows. These features explicitly document the required version.
 

source/common/event/file_event_impl.cc

@@ -3,7 +3,6 @@
 #include <cstdint>
 
 #include "source/common/common/assert.h"
-#include "source/common/event/dispatcher_impl.h"
 
 #include "event2/event.h"
 

source/extensions/extensions_build_config.bzl

@@ -260,6 +260,7 @@ EXTENSIONS = {
     #
 
     "envoy.io_socket.user_space":                       "//source/extensions/io_socket/user_space:config",
+    "envoy.io_socket.io_uring":                         "//source/extensions/io_socket/io_uring:config",
 
     #
     # TLS peer certification validators

source/extensions/extensions_metadata.yaml

@@ -473,6 +473,11 @@ envoy.internal_redirect_predicates.safe_cross_scheme:
   - envoy.internal_redirect_predicates
   security_posture: robust_to_untrusted_downstream_and_upstream
   status: stable
+envoy.io_socket.io_uring:
+  categories:
+  - envoy.io_socket
+  security_posture: unknown
+  status: wip
 envoy.io_socket.user_space:
   categories:
   - envoy.io_socket

source/extensions/io_socket/io_uring/BUILD

@@ -0,0 +1,67 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_extension",
+    "envoy_cc_library",
+    "envoy_extension_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_extension_package()
+
+envoy_cc_extension(
+    name = "config",
+    srcs = ["config.cc"],
+    hdrs = ["config.h"],
+    deps = [
+        ":io_handle_impl_lib",
+        "//source/common/network:socket_interface_lib",
+        "@envoy_api//envoy/extensions/network/socket_interface/v3:pkg_cc_proto",
+    ] + select({
+        "//bazel:linux": [":io_uring_impl_lib"],
+        "//conditions:default": [],
+    }),
+)
+
+envoy_cc_library(
+    name = "io_handle_impl_lib",
+    srcs = [
+        "io_handle_impl.cc",
+    ],
+    hdrs = [
+        "io_handle_impl.h",
+    ],
+    deps = [
+        ":io_uring_interface",
+        "//envoy/event:dispatcher_interface",
+        "//envoy/network:io_handle_interface",
+        "//source/common/api:os_sys_calls_lib",
+        "//source/common/buffer:buffer_lib",
+        "//source/common/network:address_lib",
+        "//source/common/network:io_socket_error_lib",
+    ],
+)
+
+envoy_cc_library(
+    name = "io_uring_interface",
+    hdrs = [
+        "io_uring.h",
+    ],
+)
+
+envoy_cc_library(
+    name = "io_uring_impl_lib",
+    srcs = [
+        "io_uring_impl.cc",
+    ],
+    hdrs = [
+        "io_uring_impl.h",
+    ],
+    external_deps = ["uring"],
+    visibility = ["//visibility:private"],
+    deps = [
+        ":io_uring_interface",
+        "//source/common/network:address_lib",
+        "//source/common/network:io_socket_error_lib",
+    ],
+)

source/extensions/io_socket/io_uring/config.cc

@@ -0,0 +1,125 @@
+#include "source/extensions/io_socket/io_uring/config.h"
+
+#include "envoy/extensions/network/socket_interface/v3/io_uring_socket_interface.pb.validate.h"
+
+#include "source/common/api/os_sys_calls_impl.h"
+#include "source/extensions/io_socket/io_uring/io_handle_impl.h"
+
+#if defined(__linux__)
+#include "source/extensions/io_socket/io_uring/io_uring_impl.h"
+#endif
+
+namespace Envoy {
+namespace Extensions {
+namespace IoSocket {
+namespace IoUring {
+
+namespace {
+
+const uint32_t DefaultReadBufferSize = 8192;
+const uint32_t DefaultIoUringSize = 300;
+
+} // namespace
+
+Network::IoHandlePtr
+SocketInterfaceImpl::socket(Network::Socket::Type socket_type, Network::Address::Type addr_type,
+                            Network::Address::IpVersion version, bool socket_v6only,
+                            const Network::SocketCreationOptions& options) const {
+  int protocol = 0;
+  int flags = 0;
+
+  if (options.mptcp_enabled_) {
+    ASSERT(socket_type == Network::Socket::Type::Stream);
+    ASSERT(addr_type == Network::Address::Type::Ip);
+    protocol = IPPROTO_MPTCP;
+  }
+
+  if (socket_type == Network::Socket::Type::Stream) {
+    flags |= SOCK_STREAM;
+  } else {
+    flags |= SOCK_DGRAM;
+  }
+
+  int domain;
+  if (addr_type == Network::Address::Type::Ip) {
+    if (version == Network::Address::IpVersion::v6) {
+      domain = AF_INET6;
+    } else {
+      ASSERT(version == Network::Address::IpVersion::v4);
+      domain = AF_INET;
+    }
+  } else if (addr_type == Network::Address::Type::Pipe) {
+    domain = AF_UNIX;
+  } else {
+    NOT_IMPLEMENTED_GCOVR_EXCL_LINE;
+  }
+
+  const Api::SysCallSocketResult result =
+      Api::OsSysCallsSingleton::get().socket(domain, flags, protocol);
+  RELEASE_ASSERT(SOCKET_VALID(result.return_value_),
+                 fmt::format("socket(2) failed, got error: {}", errorDetails(result.errno_)));
+
+  ASSERT(io_uring_factory_ != nullptr);
+  return std::make_unique<IoUringSocketHandleImpl>(read_buffer_size_, *io_uring_factory_,
+                                                   result.return_value_, socket_v6only, domain);
+}
+
+Network::IoHandlePtr
+SocketInterfaceImpl::socket(Network::Socket::Type socket_type,
+                            const Network::Address::InstanceConstSharedPtr addr,
+                            const Network::SocketCreationOptions& options) const {
+  Network::Address::IpVersion ip_version =
+      addr->ip() ? addr->ip()->version() : Network::Address::IpVersion::v4;
+  int v6only = 0;
+  if (addr->type() == Network::Address::Type::Ip && ip_version == Network::Address::IpVersion::v6) {
+    v6only = addr->ip()->ipv6()->v6only();
+  }
+
+  Network::IoHandlePtr io_handle =
+      SocketInterfaceImpl::socket(socket_type, addr->type(), ip_version, v6only, options);
+  if (addr->type() == Network::Address::Type::Ip && ip_version == Network::Address::IpVersion::v6) {
+    // Setting IPV6_V6ONLY restricts the IPv6 socket to IPv6 connections only.
+    const Api::SysCallIntResult result = io_handle->setOption(
+        IPPROTO_IPV6, IPV6_V6ONLY, reinterpret_cast<const char*>(&v6only), sizeof(v6only));
+    RELEASE_ASSERT(!SOCKET_FAILURE(result.return_value_), "");
+  }
+  return io_handle;
+}
+
+bool SocketInterfaceImpl::ipFamilySupported(int domain) {
+  Api::OsSysCalls& os_sys_calls = Api::OsSysCallsSingleton::get();
+  const Api::SysCallSocketResult result = os_sys_calls.socket(domain, SOCK_STREAM, 0);
+  if (SOCKET_VALID(result.return_value_)) {
+    RELEASE_ASSERT(
+        os_sys_calls.close(result.return_value_).return_value_ == 0,
+        fmt::format("Fail to close fd: response code {}", errorDetails(result.return_value_)));
+  }
+  return SOCKET_VALID(result.return_value_);
+}
+
+Server::BootstrapExtensionPtr SocketInterfaceImpl::createBootstrapExtension(
+    const Protobuf::Message& message, Server::Configuration::ServerFactoryContext& context) {
+  auto config = MessageUtil::downcastAndValidate<
+      const envoy::extensions::network::socket_interface::v3::IoUringSocketInterface&>(
+      message, context.messageValidationContext().staticValidationVisitor());
+  read_buffer_size_ =
+      PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, read_buffer_size, DefaultReadBufferSize);
+#if defined(__linux__)
+  io_uring_factory_ = std::make_unique<const IoUringFactoryImpl>(
+      PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, io_uring_size, DefaultIoUringSize),
+      config.use_submission_queue_polling());
+#endif
+  return std::make_unique<Network::SocketInterfaceExtension>(*this);
+}
+
+ProtobufTypes::MessagePtr SocketInterfaceImpl::createEmptyConfigProto() {
+  return std::make_unique<
+      envoy::extensions::network::socket_interface::v3::IoUringSocketInterface>();
+}
+
+REGISTER_FACTORY(SocketInterfaceImpl, Server::Configuration::BootstrapExtensionFactory);
+
+} // namespace IoUring
+} // namespace IoSocket
+} // namespace Extensions
+} // namespace Envoy