socket: IP_FREEBIND support for listeners and upstream connections.

bazel/repository_locations.bzl

@@ -76,7 +76,7 @@ REPOSITORY_LOCATIONS = dict(
         urls = ["https://github.com/google/protobuf/archive/v3.5.0.tar.gz"],
     ),
     envoy_api = dict(
-        commit = "e73b8264a26c909f75d38ec2c73d9f47ad1b3b57",
+        commit = "ec157d3d8b359a1bd65c22116ba4387a459cc53a",
         remote = "https://github.com/envoyproxy/data-plane-api",
     ),
     grpc_httpjson_transcoding = dict(

configs/freebind/README.md

@@ -0,0 +1,14 @@
+# Freebind testing
+
+To manually validate the `IP_FREEBIND` behavior in Envoy, you can launch Envoy with
+[freebind.yaml](freebind.yaml).
+
+The listener free bind behavior can be verified with:
+
+1. `envoy -c ./configs/freebind/freebind.yaml -l trace`
+2. `sudo ifconfig lo:1 192.168.42.1/30 up`
+3. `nc -v -l 0.0.0.0 10001`
+
+To cleanup run `sudo ifconfig lo:1 down`.
+
+TODO(htuch): Steps to verify upstream behavior.

configs/freebind/freebind.yaml

@@ -0,0 +1,41 @@
+admin:
+  access_log_path: /tmp/admin_access.log
+  address:
+    socket_address: { address: 127.0.0.1, port_value: 9901 }
+
+static_resources:
+  listeners:
+  - name: listener_0
+    address:
+      socket_address: { address: 192.168.42.1, port_value: 10000 }
+    freebind: true
+    filter_chains:
+    - filters:
+      - name: envoy.http_connection_manager
+        config:
+          stat_prefix: ingress_http
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_service
+              domains: ["*"]
+              routes:
+              - match: { prefix: "/" }
+                route: { cluster: service_local }
+          http_filters:
+          - name: envoy.router
+  clusters:
+  - name: service_local
+    connect_timeout: 30s
+    type: STATIC
+    lb_policy: ROUND_ROBIN
+    hosts:
+    - socket_address:
+        address: 127.0.0.1
+        port_value: 10001
+# TODO(htuch): Figure out how to do end-to-end testing with
+# outgoing connections and free bind.
+#    upstream_bind_config:
+#      source_address:
+#        address: 192.168.43.1
+#      freebind: true

include/envoy/api/os_sys_calls.h

@@ -63,6 +63,17 @@ class OsSysCalls {
    * @see man 2 stat
    */
   virtual int stat(const char* pathname, struct stat* buf) PURE;
+
+  /**
+   * @see man 2 setsockopt
+   */
+  virtual int setsockopt(int sockfd, int level, int optname, const void* optval,
+                         socklen_t optlen) PURE;
+
+  /**
+   * @see man 2 getsockopt
+   */
+  virtual int getsockopt(int sockfd, int level, int optname, void* optval, socklen_t* optlen) PURE;
 };
 
 typedef std::unique_ptr<OsSysCalls> OsSysCallsPtr;

include/envoy/network/listen_socket.h

@@ -55,13 +55,14 @@ class Socket {
      */
     virtual void hashKey(std::vector<uint8_t>& key) const PURE;
   };
-  typedef std::unique_ptr<Option> OptionPtr;
-  typedef std::shared_ptr<std::vector<OptionPtr>> OptionsSharedPtr;
+  typedef std::shared_ptr<const Option> OptionConstSharedPtr;
+  typedef std::vector<OptionConstSharedPtr> Options;
+  typedef std::shared_ptr<Options> OptionsSharedPtr;
 
   /**
    * Add a socket option visitor for later retrieval with options().
    */
-  virtual void addOption(OptionPtr&&) PURE;
+  virtual void addOption(const OptionConstSharedPtr&) PURE;
 
   /**
    * @return the socket options stored earlier with addOption() calls, if any.

include/envoy/server/filter_config.h

@@ -135,7 +135,7 @@ class ListenerFactoryContext : public FactoryContext {
   /**
    * Store socket options to be set on the listen socket before listening.
    */
-  virtual void addListenSocketOption(Network::Socket::OptionPtr&& option) PURE;
+  virtual void addListenSocketOption(const Network::Socket::OptionConstSharedPtr& option) PURE;
 };
 
 /**

include/envoy/upstream/cluster_manager.h

@@ -145,12 +145,10 @@ class ClusterManager {
   virtual void shutdown() PURE;
 
   /**
-   * Returns an optional source address for upstream connections to bind to.
-   *
-   * @return Network::Address::InstanceConstSharedPtr a source address to bind to or nullptr if no
-   * bind need occur.
+   * @return const envoy::api::v2::core::BindConfig& cluster manager wide bind configuration for new
+   *         upstream connections.
    */
-  virtual const Network::Address::InstanceConstSharedPtr& sourceAddress() const PURE;
+  virtual const envoy::api::v2::core::BindConfig& bindConfig() const PURE;
 
   /**
    * Return a reference to the singleton ADS provider for upstream control plane muxing of xDS. This

include/envoy/upstream/upstream.h

@@ -369,6 +369,8 @@ class ClusterInfo {
     // Use the downstream protocol (HTTP1.1, HTTP2) for upstream connections as well, if available.
     // This is used when creating connection pools.
     static const uint64_t USE_DOWNSTREAM_PROTOCOL = 0x2;
+    // Use IP_FREEBIND socket option when binding.
+    static const uint64_t FREEBIND = 0x4;
   };
 
   virtual ~ClusterInfo() {}

source/common/api/os_sys_calls_impl.cc

@@ -35,5 +35,15 @@ void* OsSysCallsImpl::mmap(void* addr, size_t length, int prot, int flags, int f
 
 int OsSysCallsImpl::stat(const char* pathname, struct stat* buf) { return ::stat(pathname, buf); }
 
+int OsSysCallsImpl::setsockopt(int sockfd, int level, int optname, const void* optval,
+                               socklen_t optlen) {
+  return ::setsockopt(sockfd, level, optname, optval, optlen);
+}
+
+int OsSysCallsImpl::getsockopt(int sockfd, int level, int optname, void* optval,
+                               socklen_t* optlen) {
+  return ::getsockopt(sockfd, level, optname, optval, optlen);
+}
+
 } // namespace Api
 } // namespace Envoy

source/common/api/os_sys_calls_impl.h

@@ -19,6 +19,8 @@ class OsSysCallsImpl : public OsSysCalls {
   int ftruncate(int fd, off_t length) override;
   void* mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) override;
   int stat(const char* pathname, struct stat* buf) override;
+  int setsockopt(int sockfd, int level, int optname, const void* optval, socklen_t optlen) override;
+  int getsockopt(int sockfd, int level, int optname, void* optval, socklen_t* optlen) override;
 };
 
 typedef ThreadSafeSingleton<OsSysCallsImpl> OsSysCallsSingleton;

source/common/network/BUILD

@@ -173,6 +173,20 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "socket_option_lib",
+    srcs = ["socket_option_impl.cc"],
+    hdrs = ["socket_option_impl.h"],
+    external_deps = ["abseil_optional"],
+    deps = [
+        ":address_lib",
+        "//include/envoy/network:listen_socket_interface",
+        "//source/common/api:os_sys_calls_lib",
+        "//source/common/common:assert_lib",
+        "//source/common/common:logger_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "utility_lib",
     srcs = ["utility.cc"],

source/common/network/listen_socket_impl.cc

@@ -45,6 +45,7 @@ TcpListenSocket::TcpListenSocket(const Address::InstanceConstSharedPtr& address,
     : ListenSocketImpl(address->socket(Address::SocketType::Stream), address) {
   RELEASE_ASSERT(fd_ != -1);
 
+  // TODO(htuch): This might benefit from moving to SocketOptionImpl.
   int on = 1;
   int rc = setsockopt(fd_, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
   RELEASE_ASSERT(rc != -1);

source/common/network/listen_socket_impl.h

@@ -26,9 +26,9 @@ class SocketImpl : public virtual Socket {
       fd_ = -1;
     }
   }
-  void addOption(OptionPtr&& option) override {
+  void addOption(const OptionConstSharedPtr& option) override {
     if (!options_) {
-      options_ = std::make_shared<std::vector<OptionPtr>>();
+      options_ = std::make_shared<std::vector<OptionConstSharedPtr>>();
     }
     options_->emplace_back(std::move(option));
   }

source/common/network/socket_option_impl.cc

@@ -0,0 +1,80 @@
+#include "common/network/socket_option_impl.h"
+
+#include "envoy/common/exception.h"
+
+#include "common/api/os_sys_calls_impl.h"
+#include "common/common/assert.h"
+#include "common/network/address_impl.h"
+
+namespace Envoy {
+namespace Network {
+
+bool SocketOptionImpl::setOption(Socket& socket, Socket::SocketState state) const {
+  if (state == Socket::SocketState::PreBind) {
+    if (freebind_.has_value()) {
+      const int should_freebind = freebind_.value() ? 1 : 0;
+      const int error =
+          setIpSocketOption(socket, ENVOY_SOCKET_IP_FREEBIND, ENVOY_SOCKET_IPV6_FREEBIND,
+                            &should_freebind, sizeof(should_freebind));
+      if (error != 0) {
+        ENVOY_LOG(warn, "Setting IP_FREEBIND on listener socket failed: {}", strerror(error));
+        return false;
+      }
+    }
+  }
+
+  return true;
+}
+
+int SocketOptionImpl::setIpSocketOption(Socket& socket, SocketOptionName ipv4_optname,
+                                        SocketOptionName ipv6_optname, const void* optval,
+                                        socklen_t optlen) {
+  auto& os_syscalls = Api::OsSysCallsSingleton::get();
+
+  // If this isn't IP, we're out of luck.
+  Address::InstanceConstSharedPtr address;
+  const Address::Ip* ip = nullptr;
+  try {
+    // We have local address when the socket is used in a listener but have to
+    // infer the IP from the socket FD when initiating connections.
+    // TODO(htuch): Figure out a way to obtain a consistent interface for IP
+    // version from socket.
+    if (socket.localAddress()) {
+      ip = socket.localAddress()->ip();
+    } else {
+      address = Address::addressFromFd(socket.fd());
+      ip = address->ip();
+    }
+  } catch (const EnvoyException&) {
+    // Ignore, we get here because we failed in getsockname().
+    // TODO(htuch): We should probably clean up this logic to avoid relying on exceptions.
+  }
+  if (ip == nullptr) {
+    ENVOY_LOG(warn, "Failed to set IP socket option on non-IP socket");
+    return ENOTSUP;
+  }
+
+  // If the FD is v4, we can only try the IPv4 variant.
+  if (ip->version() == Network::Address::IpVersion::v4) {
+    if (!ipv4_optname) {
+      ENVOY_LOG(warn, "Unsupported IPv4 socket option");
+      return ENOTSUP;
+    }
+    return os_syscalls.setsockopt(socket.fd(), IPPROTO_IP, ipv4_optname.value(), optval, optlen);
+  }
+
+  // If the FD is v6, we first try the IPv6 variant if the platfrom supports it and fallback to the
+  // IPv4 variant otherwise.
+  ASSERT(ip->version() == Network::Address::IpVersion::v6);
+  if (ipv6_optname) {
+    return os_syscalls.setsockopt(socket.fd(), IPPROTO_IPV6, ipv6_optname.value(), optval, optlen);
+  }
+  if (ipv4_optname) {
+    return os_syscalls.setsockopt(socket.fd(), IPPROTO_IP, ipv4_optname.value(), optval, optlen);
+  }
+  ENVOY_LOG(warn, "Unsupported IPv6 socket option");
+  return ENOTSUP;
+}
+
+} // namespace Network
+} // namespace Envoy

source/common/network/socket_option_impl.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <sys/socket.h>
+
+#include "envoy/network/listen_socket.h"
+
+#include "common/common/logger.h"
+
+#include "absl/types/optional.h"
+
+namespace Envoy {
+namespace Network {
+
+// Optional variant of setsockopt(2) optname. The idea here is that if the option is not supported
+// on a platform, we can make this the empty value. This allows us to avoid proliferation of #ifdef.
+typedef absl::optional<int> SocketOptionName;
+
+#ifdef IP_FREEBIND
+#define ENVOY_SOCKET_IP_FREEBIND Network::SocketOptionName(IP_FREEBIND)
+#else
+#define ENVOY_SOCKET_IP_FREEBIND Network::SocketOptionName()
+#endif
+
+#ifdef IPV6_FREEBIND
+#define ENVOY_SOCKET_IPV6_FREEBIND Network::SocketOptionName(IPV6_FREEBIND)
+#else
+#define ENVOY_SOCKET_IPV6_FREEBIND Network::SocketOptionName()
+#endif
+
+class SocketOptionImpl : public Socket::Option, Logger::Loggable<Logger::Id::connection> {
+public:
+  SocketOptionImpl(absl::optional<bool> freebind) : freebind_(freebind) {}
+
+  // Socket::Option
+  bool setOption(Socket& socket, Socket::SocketState state) const override;
+  // The common socket options don't require a hash key.
+  void hashKey(std::vector<uint8_t>&) const override {}
+
+  /**
+   * Set a socket option that applies at both IPv4 and IPv6 socket levels. When the underlying FD
+   * is IPv6, this function will attempt to set at IPv6 unless the platform only supports the
+   * option at the IPv4 level.
+   * @param socket.
+   * @param ipv4_optname SocketOptionName for IPv4 level. Set to empty if not supported on
+   * platform.
+   * @param ipv6_optname SocketOptionName for IPv6 level. Set to empty if not supported on
+   * platform.
+   * @param optval as per setsockopt(2).
+   * @param optlen as per setsockopt(2).
+   * @return int as per setsockopt(2). ENOTSUP is returned if the option is not supported on the
+   * platform for fd after the above option level fallback semantics are taken into account or the
+   *         socket is non-IP.
+   */
+  static int setIpSocketOption(Socket& socket, SocketOptionName ipv4_optname,
+                               SocketOptionName ipv6_optname, const void* optval, socklen_t optlen);
+
+private:
+  const absl::optional<bool> freebind_;
+};
+
+} // namespace Network
+} // namespace Envoy

source/common/upstream/BUILD

@@ -342,6 +342,7 @@ envoy_cc_library(
         "//source/common/http:utility_lib",
         "//source/common/network:address_lib",
         "//source/common/network:resolver_lib",
+        "//source/common/network:socket_option_lib",
         "//source/common/network:utility_lib",
         "//source/common/protobuf",
         "//source/common/protobuf:utility_lib",

source/common/upstream/cluster_manager_impl.cc

@@ -170,7 +170,8 @@ ClusterManagerImpl::ClusterManagerImpl(const envoy::config::bootstrap::v2::Boots
                                        AccessLog::AccessLogManager& log_manager,
                                        Event::Dispatcher& main_thread_dispatcher)
     : factory_(factory), runtime_(runtime), stats_(stats), tls_(tls.allocateSlot()),
-      random_(random), local_info_(local_info), cm_stats_(generateStats(stats)),
+      random_(random), bind_config_(bootstrap.cluster_manager().upstream_bind_config()),
+      local_info_(local_info), cm_stats_(generateStats(stats)),
       init_helper_([this](Cluster& cluster) { onClusterInit(cluster); }) {
   async_client_manager_ = std::make_unique<Grpc::AsyncClientManagerImpl>(*this, tls);
   const auto& cm_config = bootstrap.cluster_manager();
@@ -187,11 +188,6 @@ ClusterManagerImpl::ClusterManagerImpl(const envoy::config::bootstrap::v2::Boots
     eds_config_ = bootstrap.dynamic_resources().deprecated_v1().sds_config();
   }
 
-  if (bootstrap.cluster_manager().upstream_bind_config().has_source_address()) {
-    source_address_ = Network::Address::resolveProtoSocketAddress(
-        bootstrap.cluster_manager().upstream_bind_config().source_address());
-  }
-
   // Cluster loading happens in two phases: first all the primary clusters are loaded, and then all
   // the secondary clusters are loaded. As it currently stands all non-EDS clusters are primary and
   // only EDS clusters are secondary. This two phase loading is done because in v2 configuration

source/common/upstream/cluster_manager_impl.h

@@ -181,9 +181,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable<Logger::Id::u
     active_clusters_.clear();
   }
 
-  const Network::Address::InstanceConstSharedPtr& sourceAddress() const override {
-    return source_address_;
-  }
+  const envoy::api::v2::core::BindConfig& bindConfig() const override { return bind_config_; }
 
   Config::GrpcMux& adsMux() override { return *ads_mux_; }
   Grpc::AsyncClientManager& grpcAsyncClientManager() override { return *async_client_manager_; }
@@ -309,7 +307,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable<Logger::Id::u
   ClusterMap active_clusters_;
   ClusterMap warming_clusters_;
   absl::optional<envoy::api::v2::core::ConfigSource> eds_config_;
-  Network::Address::InstanceConstSharedPtr source_address_;
+  envoy::api::v2::core::BindConfig bind_config_;
   Outlier::EventLoggerSharedPtr outlier_event_logger_;
   const LocalInfo::LocalInfo& local_info_;
   CdsApiPtr cds_api_;