upstream: add transparency options

api/envoy/api/v2/cds.proto

@@ -558,4 +558,13 @@ message UpstreamBindConfig {
 message UpstreamConnectionOptions {
   // If set then set SO_KEEPALIVE on the socket to enable TCP Keepalives.
   core.TcpKeepalive tcp_keepalive = 1;
+
+  // If true, the source address of the origin is spoofed on the connection to the upstream.
+  // [#not-implemented-hide:] Hide from docs.
+  bool src_transparent = 2;
+
+  // If non-zero, set the SO_MARK option on the upstream connection's socket. Allows matching on
+  // e.g. 'ip rule' or 'iptables'.
+  // [#proto-status: draft] Still working on this.
+  uint32 mark = 3;
 }

source/common/network/socket_option_factory.cc

@@ -49,6 +49,15 @@ std::unique_ptr<Socket::Options> SocketOptionFactory::buildIpTransparentOptions(
   return options;
 }
 
+std::unique_ptr<Socket::Options> SocketOptionFactory::buildSocketMarkOptions(uint32_t mark) {
+  std::unique_ptr<Socket::Options> options = absl::make_unique<Socket::Options>();
+  // we need this to happen prior to binding or prior to connecting. In both cases, PREBIND will
+  // fire.
+  options->push_back(std::make_shared<Network::SocketOptionImpl>(
+      envoy::api::v2::core::SocketOption::STATE_PREBIND, ENVOY_SOCKET_SO_MARK, mark));
+  return options;
+}
+
 std::unique_ptr<Socket::Options> SocketOptionFactory::buildLiteralOptions(
     const Protobuf::RepeatedPtrField<envoy::api::v2::core::SocketOption>& socket_options) {
   auto options = absl::make_unique<Socket::Options>();

source/common/network/socket_option_factory.h

@@ -29,6 +29,7 @@ class SocketOptionFactory : Logger::Loggable<Logger::Id::connection> {
   buildTcpKeepaliveOptions(Network::TcpKeepaliveConfig keepalive_config);
   static std::unique_ptr<Socket::Options> buildIpFreebindOptions();
   static std::unique_ptr<Socket::Options> buildIpTransparentOptions();
+  static std::unique_ptr<Socket::Options> buildSocketMarkOptions(uint32_t mark);
   static std::unique_ptr<Socket::Options> buildTcpFastOpenOptions(uint32_t queue_length);
   static std::unique_ptr<Socket::Options> buildLiteralOptions(
       const Protobuf::RepeatedPtrField<envoy::api::v2::core::SocketOption>& socket_options);

source/common/network/socket_option_impl.h

@@ -53,6 +53,12 @@ typedef absl::optional<std::pair<int, int>> SocketOptionName;
 #define ENVOY_SOCKET_SO_KEEPALIVE Network::SocketOptionName()
 #endif
 
+#ifdef SO_MARK
+#define ENVOY_SOCKET_SO_MARK Network::SocketOptionName(std::make_pair(SOL_SOCKET, SO_MARK))
+#else
+#define ENVOY_SOCKET_SO_MARK Network::SocketOptionName()
+#endif
+
 #ifdef TCP_KEEPCNT
 #define ENVOY_SOCKET_TCP_KEEPCNT Network::SocketOptionName(std::make_pair(IPPROTO_TCP, TCP_KEEPCNT))
 #else

source/common/upstream/upstream_impl.cc

@@ -93,6 +93,15 @@ parseClusterSocketOptions(const envoy::api::v2::Cluster& config,
     Network::Socket::appendOptions(cluster_options,
                                    Network::SocketOptionFactory::buildIpFreebindOptions());
   }
+  if (config.upstream_connection_options().src_transparent()) {
+    Network::Socket::appendOptions(cluster_options,
+                                   Network::SocketOptionFactory::buildIpTransparentOptions());
+  }
+  const uint32_t mark = config.upstream_connection_options().mark();
+  if (mark != 0) {
+    Network::Socket::appendOptions(cluster_options,
+                                   Network::SocketOptionFactory::buildSocketMarkOptions(mark));
+  }
   if (config.upstream_connection_options().has_tcp_keepalive()) {
     Network::Socket::appendOptions(
         cluster_options,

test/common/network/BUILD

@@ -182,6 +182,20 @@ envoy_cc_test(
     ],
 )
 
+envoy_cc_test(
+    name = "socket_option_factory_test",
+    srcs = ["socket_option_factory_test.cc"],
+    deps = [
+        "//source/common/network:address_lib",
+        "//source/common/network:socket_option_factory_lib",
+        "//source/common/network:socket_option_lib",
+        "//test/mocks/api:api_mocks",
+        "//test/mocks/network:network_mocks",
+        "//test/test_common:environment_lib",
+        "//test/test_common:threadsafe_singleton_injector_lib",
+    ],
+)
+
 envoy_cc_test(
     name = "addr_family_aware_socket_option_impl_test",
     srcs = ["addr_family_aware_socket_option_impl_test.cc"],

test/common/network/socket_option_factory_test.cc

@@ -0,0 +1,124 @@
+#include "common/network/address_impl.h"
+#include "common/network/socket_option_factory.h"
+#include "common/network/socket_option_impl.h"
+
+#include "test/mocks/api/mocks.h"
+#include "test/mocks/network/mocks.h"
+#include "test/test_common/threadsafe_singleton_injector.h"
+
+#include "gtest/gtest.h"
+
+using testing::_;
+
+namespace Envoy {
+namespace Network {
+
+class SocketOptionFactoryTest : public testing::Test {
+public:
+  SocketOptionFactoryTest() = default;
+
+  TestThreadsafeSingletonInjector<Api::OsSysCallsImpl> os_calls_{[this]() {
+    // Before injecting OsSysCallsImpl, make sure validateIpv{4,6}Supported is called so the static
+    // bool is initialized without requiring to mock ::socket and ::close. :( :(
+    std::make_unique<Address::Ipv4Instance>("1.2.3.4", 5678);
+    std::make_unique<Address::Ipv6Instance>("::1:2:3:4", 5678);
+    return &os_sys_calls_mock_;
+  }()};
+
+protected:
+  testing::NiceMock<MockListenSocket> socket_mock_;
+  Api::MockOsSysCalls os_sys_calls_mock_;
+
+  void SetUp() override { socket_mock_.local_address_.reset(); }
+  void makeSocketV4() {
+    socket_mock_.local_address_ = std::make_unique<Address::Ipv4Instance>("1.2.3.4", 5678);
+  }
+  void makeSocketV6() {
+    socket_mock_.local_address_ = std::make_unique<Address::Ipv6Instance>("::1:2:3:4", 5678);
+  }
+};
+
+#define CHECK_OPTION_SUPPORTED(option)                                                             \
+  if (!option.has_value()) {                                                                       \
+    return;                                                                                        \
+  }
+
+TEST_F(SocketOptionFactoryTest, TestBuildSocketMarkOptions) {
+
+  // use a shared_ptr due to applyOptions requiring one
+  std::shared_ptr<Socket::Options> options = SocketOptionFactory::buildSocketMarkOptions(100);
+
+  const auto expected_option = ENVOY_SOCKET_SO_MARK;
+  CHECK_OPTION_SUPPORTED(expected_option);
+
+  const int type = expected_option.value().first;
+  const int option = expected_option.value().second;
+  EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int)))
+      .WillOnce(Invoke([type, option](int, int input_type, int input_option, const void* optval,
+                                      socklen_t) -> int {
+        EXPECT_EQ(100, *static_cast<const int*>(optval));
+        EXPECT_EQ(type, input_type);
+        EXPECT_EQ(option, input_option);
+        return 0;
+      }));
+
+  EXPECT_TRUE(Network::Socket::applyOptions(options, socket_mock_,
+                                            envoy::api::v2::core::SocketOption::STATE_PREBIND));
+}
+
+TEST_F(SocketOptionFactoryTest, TestBuildIpv4TransparentOptions) {
+  makeSocketV4();
+
+  // use a shared_ptr due to applyOptions requiring one
+  std::shared_ptr<Socket::Options> options = SocketOptionFactory::buildIpTransparentOptions();
+
+  const auto expected_option = ENVOY_SOCKET_IP_TRANSPARENT;
+  CHECK_OPTION_SUPPORTED(expected_option);
+
+  const int type = expected_option.value().first;
+  const int option = expected_option.value().second;
+  EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int)))
+      .Times(2)
+      .WillRepeatedly(Invoke([type, option](int, int input_type, int input_option,
+                                            const void* optval, socklen_t) -> int {
+        EXPECT_EQ(type, input_type);
+        EXPECT_EQ(option, input_option);
+        EXPECT_EQ(1, *static_cast<const int*>(optval));
+        return 0;
+      }));
+
+  EXPECT_TRUE(Network::Socket::applyOptions(options, socket_mock_,
+                                            envoy::api::v2::core::SocketOption::STATE_PREBIND));
+  EXPECT_TRUE(Network::Socket::applyOptions(options, socket_mock_,
+                                            envoy::api::v2::core::SocketOption::STATE_BOUND));
+}
+
+TEST_F(SocketOptionFactoryTest, TestBuildIpv6TransparentOptions) {
+  makeSocketV6();
+
+  // use a shared_ptr due to applyOptions requiring one
+  std::shared_ptr<Socket::Options> options = SocketOptionFactory::buildIpTransparentOptions();
+
+  const auto expected_option = ENVOY_SOCKET_IPV6_TRANSPARENT;
+  CHECK_OPTION_SUPPORTED(expected_option);
+
+  const int type = expected_option.value().first;
+  const int option = expected_option.value().second;
+  EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int)))
+      .Times(2)
+      .WillRepeatedly(Invoke([type, option](int, int input_type, int input_option,
+                                            const void* optval, socklen_t) -> int {
+        EXPECT_EQ(type, input_type);
+        EXPECT_EQ(option, input_option);
+        EXPECT_EQ(1, *static_cast<const int*>(optval));
+        return 0;
+      }));
+
+  EXPECT_TRUE(Network::Socket::applyOptions(options, socket_mock_,
+                                            envoy::api::v2::core::SocketOption::STATE_PREBIND));
+  EXPECT_TRUE(Network::Socket::applyOptions(options, socket_mock_,
+                                            envoy::api::v2::core::SocketOption::STATE_BOUND));
+}
+
+} // namespace Network
+} // namespace Envoy

test/common/upstream/BUILD

@@ -355,18 +355,22 @@ envoy_cc_test(
         "//source/common/event:dispatcher_lib",
         "//source/common/json:config_schemas_lib",
         "//source/common/json:json_loader_lib",
+        "//source/common/network:address_lib",
+        "//source/common/network:socket_option_lib",
         "//source/common/network:utility_lib",
         "//source/common/upstream:upstream_includes",
         "//source/common/upstream:upstream_lib",
         "//source/extensions/transport_sockets/raw_buffer:config",
         "//source/server:transport_socket_config_lib",
         "//test/mocks:common_lib",
+        "//test/mocks/api:api_mocks",
         "//test/mocks/local_info:local_info_mocks",
         "//test/mocks/network:network_mocks",
         "//test/mocks/runtime:runtime_mocks",
         "//test/mocks/ssl:ssl_mocks",
         "//test/mocks/upstream:upstream_mocks",
         "//test/test_common:registry_lib",
+        "//test/test_common:threadsafe_singleton_injector_lib",
         "//test/test_common:utility_lib",
     ],
 )

test/common/upstream/upstream_impl_test.cc

@@ -13,19 +13,23 @@
 #include "common/config/metadata.h"
 #include "common/json/config_schemas.h"
 #include "common/json/json_loader.h"
+#include "common/network/address_impl.h"
+#include "common/network/socket_option_impl.h"
 #include "common/network/utility.h"
 #include "common/upstream/upstream_impl.h"
 
 #include "server/transport_socket_config_impl.h"
 
 #include "test/common/upstream/utility.h"
+#include "test/mocks/api/mocks.h"
 #include "test/mocks/common.h"
 #include "test/mocks/local_info/mocks.h"
 #include "test/mocks/network/mocks.h"
 #include "test/mocks/runtime/mocks.h"
 #include "test/mocks/ssl/mocks.h"
 #include "test/mocks/upstream/mocks.h"
 #include "test/test_common/registry.h"
+#include "test/test_common/threadsafe_singleton_injector.h"
 #include "test/test_common/utility.h"
 
 #include "gmock/gmock.h"
@@ -1557,6 +1561,11 @@ TEST(PrioritySet, Extend) {
   }
 }
 
+#define CHECK_OPTION_SUPPORTED(option)                                                             \
+  if (!option.has_value()) {                                                                       \
+    return;                                                                                        \
+  }
+
 class ClusterInfoImplTest : public testing::Test {
 public:
   std::unique_ptr<StrictDnsClusterImpl> makeCluster(const std::string& yaml) {
@@ -1583,6 +1592,18 @@ class ClusterInfoImplTest : public testing::Test {
   envoy::api::v2::Cluster cluster_config_;
   Envoy::Stats::ScopePtr scope_;
   std::unique_ptr<Server::Configuration::TransportSocketFactoryContextImpl> factory_context_;
+
+protected:
+  TestThreadsafeSingletonInjector<Api::OsSysCallsImpl> os_calls_{[this]() {
+    // Before injecting OsSysCallsImpl, make sure validateIpv{4,6}Supported is called so the static
+    // bool is initialized without requiring to mock ::socket and ::close. :( :(
+    std::make_unique<Envoy::Network::Address::Ipv4Instance>("1.2.3.4", 5678);
+    std::make_unique<Envoy::Network::Address::Ipv6Instance>("::1:2:3:4", 5678);
+    return &os_sys_calls_mock_;
+  }()};
+
+  testing::NiceMock<Envoy::Network::MockListenSocket> socket_mock_;
+  Api::MockOsSysCalls os_sys_calls_mock_;
 };
 
 struct Foo : public Envoy::Config::TypedMetadata::Object {};
@@ -1838,6 +1859,65 @@ TEST_F(ClusterInfoImplTest, ExtensionProtocolOptionsForFilterWithOptions) {
   }
 }
 
+TEST_F(ClusterInfoImplTest, TestNoSocketOptionsDefault) {
+  auto cluster = makeCluster(strictDnsClusterWithAttrYaml("" /* no extra options */));
+  auto options = cluster->info()->clusterSocketOptions();
+
+  EXPECT_EQ(nullptr, options);
+}
+
+TEST_F(ClusterInfoImplTest, TestMarkSocketOptions) {
+  auto cluster = makeCluster(strictDnsClusterWithAttrYaml(R"EOF(
+    upstream_connection_options:
+      mark: 145
+  )EOF"));
+  auto options = cluster->info()->clusterSocketOptions();
+
+  const auto expected_option = ENVOY_SOCKET_SO_MARK;
+  CHECK_OPTION_SUPPORTED(expected_option);
+  const int type = expected_option.value().first;
+  const int option = expected_option.value().second;
+
+  EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int)))
+      .WillOnce(Invoke([type, option](int, int input_type, int input_option, const void* optval,
+                                      socklen_t) -> int {
+        EXPECT_EQ(145, *static_cast<const int*>(optval));
+        EXPECT_EQ(type, input_type);
+        EXPECT_EQ(option, input_option);
+        return 0;
+      }));
+
+  Envoy::Network::Socket::applyOptions(options, socket_mock_,
+                                       envoy::api::v2::core::SocketOption::STATE_PREBIND);
+}
+
+TEST_F(ClusterInfoImplTest, TestTransparentSocketOptions) {
+  auto cluster = makeCluster(strictDnsClusterWithAttrYaml(R"EOF(
+    upstream_connection_options:
+      src_transparent: true
+  )EOF"));
+  auto options = cluster->info()->clusterSocketOptions();
+
+  const auto expected_option = ENVOY_SOCKET_IP_TRANSPARENT;
+  CHECK_OPTION_SUPPORTED(expected_option);
+  const int type = expected_option.value().first;
+  const int option = expected_option.value().second;
+  socket_mock_.local_address_ =
+      std::make_unique<Envoy::Network::Address::Ipv4Instance>("1.2.3.4", 5678);
+
+  EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int)))
+      .WillOnce(Invoke([type, option](int, int input_type, int input_option, const void* optval,
+                                      socklen_t) -> int {
+        EXPECT_EQ(type, input_type);
+        EXPECT_EQ(option, input_option);
+        EXPECT_EQ(1, *static_cast<const int*>(optval));
+        return 0;
+      }));
+
+  Envoy::Network::Socket::applyOptions(options, socket_mock_,
+                                       envoy::api::v2::core::SocketOption::STATE_PREBIND);
+}
+
 // Validate empty singleton for HostsPerLocalityImpl.
 TEST(HostsPerLocalityImpl, Empty) {
   EXPECT_FALSE(HostsPerLocalityImpl::empty()->hasLocalLocality());