Add support for multiple sds configs in DownstramTlsContext

api/envoy/extensions/transport_sockets/tls/v3/tls.proto

@@ -254,8 +254,7 @@ message CommonTlsContext {
   // [#next-major-version: These mutually exclusive fields should ideally be in a oneof, but it's
   // not legal to put a repeated field in a oneof. In the next major version, we should rework
   // this to avoid this problem.]
-  repeated SdsSecretConfig tls_certificate_sds_secret_configs = 6
-      [(validate.rules).repeated = {max_items: 2}];
+  repeated SdsSecretConfig tls_certificate_sds_secret_configs = 6;
 
   // Certificate provider instance for fetching TLS certs.
   //

test/config/integration/certs/certs.sh

@@ -68,6 +68,8 @@ generate_info_header() {
         echo "constexpr char ${prefix}_CERT_1_HASH[] = \"$(openssl x509 -in "${1}cert.pem" -outform DER | openssl dgst -sha1 | cut -d" " -f2)\";"
         echo "constexpr char ${prefix}_CERT_SPKI[] = \"$(openssl x509 -in "${1}cert.pem" -noout -pubkey | openssl pkey -pubin -outform DER | openssl dgst -sha256 -binary | openssl enc -base64)\";"
         echo "constexpr char ${prefix}_CERT_SERIAL[] = \"$(openssl x509 -in "${1}cert.pem" -noout -serial | cut -d"=" -f2 | awk '{print tolower($0)}')\";"
+        echo "constexpr char ${prefix}_CERT_NOT_BEFORE[] = \"$(openssl x509 -in "${1}cert.pem" -noout -startdate | cut -d"=" -f2)\";"
+        echo "constexpr char ${prefix}_CERT_NOT_AFTER[] = \"$(openssl x509 -in "${1}cert.pem" -noout -enddate | cut -d"=" -f2)\";"
     } > "${1}cert_info.h"
 }
 
@@ -85,6 +87,11 @@ cat intermediate_cacert.pem intermediate_ca_2cert.pem > intermediate_partial_ca_
 generate_rsa_key server ca
 generate_x509_cert server ca
 generate_ocsp_response server ca
+generate_info_header server
+# Generate RSA cert for the server with different SAN
+generate_rsa_key server2 ca
+generate_x509_cert server2 ca
+generate_info_header server2
 # Generate ECDSA cert for the server.
 cp -f servercert.cfg server_ecdsacert.cfg
 generate_ecdsa_key server_ecdsa ca

test/config/integration/certs/server2cert.cfg

@@ -0,0 +1,41 @@
+[req]
+distinguished_name = req_distinguished_name
+req_extensions = v3_req
+
+[req_distinguished_name]
+countryName = US
+countryName_default = US
+stateOrProvinceName = California
+stateOrProvinceName_default = California
+localityName = San Francisco
+localityName_default = San Francisco
+organizationName = Lyft
+organizationName_default = Lyft
+organizationalUnitName = Lyft Engineering
+organizationalUnitName_default = Lyft Engineering
+commonName = Test Backend Team
+commonName_default = Test Backend Team
+commonName_max  = 64
+emailAddress = backend-team@lyft.com
+emailAddress_default = backend-team@lyft.com
+
+[v3_req]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth, serverAuth
+subjectAltName = @alt_names
+subjectKeyIdentifier = hash
+
+[v3_ca]
+basicConstraints = critical, CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth, serverAuth
+subjectAltName = @alt_names
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always
+
+[alt_names]
+URI.1 = spiffe://lyft.com/backend-team
+URI.2 = http://backend.lyft.com
+DNS.1 = lyft2.com
+DNS.2 = www.lyft2.com

test/config/integration/certs/server2cert.pem

@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIEhzCCA2+gAwIBAgIUP0pvp6i48a1geD54z7MUaSOZiI4wDQYJKoZIhvcNAQEL
+BQAwdjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM
+DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n
+aW5lZXJpbmcxEDAOBgNVBAMMB1Rlc3QgQ0EwHhcNMjMwMTEyMDMzNTE4WhcNMjUw
+MTExMDMzNTE4WjCBpjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx
+FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsM
+EEx5ZnQgRW5naW5lZXJpbmcxGjAYBgNVBAMMEVRlc3QgQmFja2VuZCBUZWFtMSQw
+IgYJKoZIhvcNAQkBFhViYWNrZW5kLXRlYW1AbHlmdC5jb20wggEiMA0GCSqGSIb3
+DQEBAQUAA4IBDwAwggEKAoIBAQC/k7DASyUBfPAVIiVHP1V1SlIrEthU3Ak8KqE3
+FFRJefWR/Du59uMd+Q0Zy4Yv01tH47DPVepzHHKBx/9Mj7PZVGfHk/hXflkPsvqj
+DkaRiajTlSSXOSDjlqOFqHf0zgw/UPtvlw9hefGJmlB+yGKUKXgU/Y079a8Pkfys
+U0zf4RwJ7puu/RMGzTcsB8eSE/oHvV5/ar7IXhCTU8tILimLSRnfN8heHzFl1Pg1
+xNL3IuMgM71JVzM02J26rM2/3Mc9Ma9Rqyvkr4g+cn3u00gLqwelvMzZErxWVO8p
+kC2toW1GJSjKccUZR5cPXO8qtPYPWrmJfLV3LVt0C+POXHQnAgMBAAGjgdswgdgw
+DAYDVR0TAQH/BAIwADALBgNVHQ8EBAMCBeAwHQYDVR0lBBYwFAYIKwYBBQUHAwIG
+CCsGAQUFBwMBMFwGA1UdEQRVMFOGHnNwaWZmZTovL2x5ZnQuY29tL2JhY2tlbmQt
+dGVhbYYXaHR0cDovL2JhY2tlbmQubHlmdC5jb22CCWx5ZnQyLmNvbYINd3d3Lmx5
+ZnQyLmNvbTAdBgNVHQ4EFgQUx0rD8uUklrtn3hhlsA2MmOppcFMwHwYDVR0jBBgw
+FoAUHQ05mHTs+06sDKaDpYtw5o+/MoIwDQYJKoZIhvcNAQELBQADggEBADGB8seV
+lshkhCrF6b/+UlHsVhA0zaT5ReaSebpyFmVDdqCVqb9TXwmLsYVPmdANmSrNt4/8
+ctk+SDRRLl0pz+ciy9d2Lw0pcF4wuy4WQFunCwSjO30H47k4GTx6AhTNk8tLBQOa
+K6FysykLmNz5Dr5jijPtOfnlpIDX0cBduAYAHX6BSkkYmyXVoAj0Ln+mBjsGCvq5
+GeGB6hRQomLcTFyBi4dj2w4nYY/XKDDa8j9C+MKDRgsE8+QxV/mKLzSsJyrZR2Xz
+KY3zVbGwMNS/4fJOxX5e71ajZRRDznrJl8ZUAiYT1dLnPqsW20DL8ARlFH1+4pML
+nam4zqTUbaBaasA=
+-----END CERTIFICATE-----

test/config/integration/certs/server2cert_hash.h

@@ -0,0 +1,3 @@
+// NOLINT(namespace-envoy)
+constexpr char TEST_SERVER2_CERT_HASH[] = "0E:A1:1B:78:09:0B:D1:D6:F4:4A:8E:D2:48:7F:C3:B8:06:A7:"
+                                          "4C:8C:E4:7A:60:3D:3F:15:B0:99:03:11:2B:20";

test/config/integration/certs/server2cert_info.h

@@ -0,0 +1,8 @@
+// NOLINT(namespace-envoy)
+constexpr char TEST_SERVER2_CERT_256_HASH[] =
+    "0ea11b78090bd1d6f44a8ed2487fc3b806a74c8ce47a603d3f15b09903112b20";
+constexpr char TEST_SERVER2_CERT_1_HASH[] = "78912aeddf93afb00f2348dc13d1edabc2e71b2c";
+constexpr char TEST_SERVER2_CERT_SPKI[] = "J/kyBd/otG9+t94S1SbU3jj4lMyjNLvUQHZK2T/8Lbs=";
+constexpr char TEST_SERVER2_CERT_SERIAL[] = "3f4a6fa7a8b8f1ad60783e78cfb314692399888e";
+constexpr char TEST_SERVER2_CERT_NOT_BEFORE[] = "Jan 12 03:35:18 2023 GMT";
+constexpr char TEST_SERVER2_CERT_NOT_AFTER[] = "Jan 11 03:35:18 2025 GMT";

test/config/integration/certs/server2key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAv5OwwEslAXzwFSIlRz9VdUpSKxLYVNwJPCqhNxRUSXn1kfw7
+ufbjHfkNGcuGL9NbR+Owz1Xqcxxygcf/TI+z2VRnx5P4V35ZD7L6ow5GkYmo05Uk
+lzkg45ajhah39M4MP1D7b5cPYXnxiZpQfshilCl4FP2NO/WvD5H8rFNM3+EcCe6b
+rv0TBs03LAfHkhP6B71ef2q+yF4Qk1PLSC4pi0kZ3zfIXh8xZdT4NcTS9yLjIDO9
+SVczNNiduqzNv9zHPTGvUasr5K+IPnJ97tNIC6sHpbzM2RK8VlTvKZAtraFtRiUo
+ynHFGUeXD1zvKrT2D1q5iXy1dy1bdAvjzlx0JwIDAQABAoIBAD9vozZ5a36LpWAK
+F3f5I84b1wuGSPYIilJO92UqqSJPbR5y/D/+3YO+RTVGbF+HyBEM9y4gj+qf/9az
+p/jtGKudRGQUvkYSg2EsbcvyXTGx+KJfH2enthlGd0051MbON8X7hdaUmbY4T9+1
+pnPk3Kec97NTpiG1n8szFwzk5G65I/SSIziB63gFsf9eIZLdSF85fY7kyWVhl39E
+loea+3FXekolIvjxh0Ui9MSzAV+0GU89wKcRp4w+reStOKkNC1RWkewMcrBPvXlI
+WHBtPpKpbe+rxwmDfFs54RLcX+8hzoe+e7DbcOkl1Ear6R9LNBYjtrPW+jU2+OAp
+Ry0oiDECgYEA5TpeNEOxkngmlE22dSyOVY7Tx01FUnC4KBn5SN+BwLnx+iCsdkHl
+TVdh70vaRjpGNZLaMXoPkV3Eq3lqbpIYiireFRMHFJ6b2B7MgeDH2pMyrFoufYxX
+MwkX38OG1U6kdAdkGNRfrEG1rvphb4FJb27RE9JC/93+1ucVYMHFF4kCgYEA1fOa
+FGj1IIholdsehqftZD7Bo0Vq/CFZGe1rkd8/bli4WEpRdOLORGFgnw3mFvmM6Jcv
+/9sYspq8M21sus/wbKqumWBQ8GsT/EVgVxe1uYNeyUA8BP0rZOODh7FIRWtpxEWA
+b1hh/Kok2ltD0JizRNU5XGSL01jsSkL45xR5ki8CgYEAjEu4Aob39Kxi+FvD811M
+1CKxjYbGidmBbNHQ2AAr2vgzmKJMy6gzHq3/u8USF+9srzdtIeESDCd2ynhqYrg6
+Gr535DAcIkudGJ5pDoiz6Rw0ZHZhg+fUKuFC4mo6aO1UC8vGQMgisjwZZbnKzsMz
+XedWQapr5UxKv975H5oY9fkCgYEAu+CU90NLrmva2211mu2v6w2o4QU+c3WdVsVE
+Zu5SN0Y1a2KnsSrR0v5n6IQ9/wLZGA5bmiP9xilXBxoaFtou2F3xwMHxkY2WGTua
+5B/v9p5ECBoeGotvJcMcZ3Xobv1p/W7C4AQO199ZSV5HaNAO7G0JV/b7SnaV2IiD
+Jfi24QUCgYBwnSwSMu029ujRVwrDzs0NOj3scpoFti9sKWjsm4ljG7DtHBTHz1fQ
+CUvsG910WCw77njpx1DCbXmDL+kdhT4TRy2cgdGGRvpn8Od12n6pF0skzz2GMkQZ
++cmiewbDUUJbjehloWJ+OxHBQwEL3jwgSFMZZcvnP9D1CAcyLQJ0+w==
+-----END RSA PRIVATE KEY-----

test/config/integration/certs/servercert_info.h

@@ -1,5 +1,8 @@
 // NOLINT(namespace-envoy)
-constexpr char TEST_SERVER_CERT_HASH[] = "DC:E2:2B:65:90:43:9A:36:1C:8E:6D:CA:42:8A:8C:37:C7:A1:77:"
-                                         "00:5B:C1:3E:33:8A:B9:2D:04:2C:B1:3F:0A";
-constexpr char TEST_SERVER_CERT_NOT_BEFORE[] = "Apr 7 16:46:35 2022 GMT";
-constexpr char TEST_SERVER_CERT_NOT_AFTER[] = "Apr 6 16:46:35 2024 GMT";
+constexpr char TEST_SERVER_CERT_256_HASH[] =
+    "dce22b6590439a361c8e6dca428a8c37c7a177005bc13e338ab92d042cb13f0a";
+constexpr char TEST_SERVER_CERT_1_HASH[] = "c777412ff69717898a3ffc61358094b6d431055a";
+constexpr char TEST_SERVER_CERT_SPKI[] = "E4cAEJmJCuF+bG3vK9LvNUaZ3Z8g+kcRKvQoJwplWAY=";
+constexpr char TEST_SERVER_CERT_SERIAL[] = "4fd5b37b416fc3fa4cbea0263c98e50fb1cd8d8e";
+constexpr char TEST_SERVER_CERT_NOT_BEFORE[] = "Apr  7 16:46:35 2022 GMT";
+constexpr char TEST_SERVER_CERT_NOT_AFTER[] = "Apr  6 16:46:35 2024 GMT";

test/extensions/transport_sockets/tls/context_impl_test.cc

@@ -1805,16 +1805,28 @@ TEST_F(ServerContextConfigImplTest, TlsCertificatesAndSdsConfig) {
       "SDS and non-SDS TLS certificates may not be mixed in server contexts");
 }
 
-TEST_F(ServerContextConfigImplTest, MultiSdsConfig) {
+TEST_F(ServerContextConfigImplTest, SdsConfigNoName) {
   envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context;
   tls_context.mutable_common_tls_context()->add_tls_certificate_sds_secret_configs();
-  tls_context.mutable_common_tls_context()->add_tls_certificate_sds_secret_configs();
   EXPECT_THROW_WITH_REGEX(
       TestUtility::validate<envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext>(
           tls_context),
       EnvoyException, "Proto constraint validation failed");
 }
 
+TEST_F(ServerContextConfigImplTest, MultiSdsConfig) {
+  envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context;
+  tls_context.mutable_common_tls_context()->add_tls_certificate_sds_secret_configs()->set_name(
+      "server_cert1");
+  tls_context.mutable_common_tls_context()->add_tls_certificate_sds_secret_configs()->set_name(
+      "server_cert2");
+  tls_context.mutable_common_tls_context()->add_tls_certificate_sds_secret_configs()->set_name(
+      "server_cert3");
+  EXPECT_NO_THROW(
+      TestUtility::validate<envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext>(
+          tls_context));
+}
+
 TEST_F(ServerContextConfigImplTest, SecretNotReady) {
   envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context;
   NiceMock<LocalInfo::MockLocalInfo> local_info;

test/integration/BUILD

@@ -1405,6 +1405,7 @@ envoy_cc_test(
         "//source/extensions/transport_sockets/tls:context_config_lib",
         "//source/extensions/transport_sockets/tls:context_lib",
         "//test/common/grpc:grpc_client_integration_lib",
+        "//test/config/integration/certs:certs_info",
         "//test/extensions/transport_sockets/tls:test_private_key_method_provider_test_lib",
         "//test/mocks/runtime:runtime_mocks",
         "//test/mocks/secret:secret_mocks",

test/integration/sds_dynamic_integration_test.cc

@@ -25,6 +25,8 @@
 
 #include "test/common/grpc/grpc_client_integration.h"
 #include "test/config/integration/certs/clientcert_hash.h"
+#include "test/config/integration/certs/servercert_info.h"
+#include "test/config/integration/certs/server2cert_info.h"
 #include "test/extensions/transport_sockets/tls/test_private_key_method_provider.h"
 #include "test/integration/http_integration.h"
 #include "test/integration/server.h"
@@ -197,6 +199,7 @@ class SdsDynamicIntegrationBaseTest : public Grpc::BaseGrpcClientIntegrationPara
   }
 
   const std::string server_cert_rsa_{"server_cert_rsa"};
+  const std::string server2_cert_rsa_{"server2_cert_rsa"};
   const std::string server_cert_ecdsa_{"server_cert_ecdsa"};
   const std::string validation_secret_{"validation_secret"};
   const std::string client_cert_{"client_cert"};
@@ -254,7 +257,7 @@ class SdsDynamicDownstreamIntegrationTest : public SdsDynamicIntegrationBaseTest
 
     // Add an additional SDS config for an EC cert (the base test has SDS config for an RSA cert).
     // This is done via the filesystem instead of gRPC to simplify the test setup.
-    if (dual_cert_) {
+    if (dual_cert_ || multi_cert_) {
       auto* secret_config_ecdsa = common_tls_context.add_tls_certificate_sds_secret_configs();
 
       secret_config_ecdsa->set_name(server_cert_ecdsa_);
@@ -283,6 +286,38 @@ version_info: "0"
       config_source->mutable_path_config_source()->set_path(sds_path);
       config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3);
     }
+
+    // Add one more additional SDS config to test multiple(>2) certs.
+    // This is done via the filesystem instead of gRPC to simplify the test setup.
+    if (multi_cert_) {
+      auto* secret_config_rsa_2 = common_tls_context.add_tls_certificate_sds_secret_configs();
+
+      secret_config_rsa_2->set_name(server2_cert_rsa_);
+      auto* config_source = secret_config_rsa_2->mutable_sds_config();
+      const std::string sds_template =
+          R"EOF(
+---
+version_info: "0"
+resources:
+- "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret
+  name: "{}"
+  tls_certificate:
+    certificate_chain:
+      filename: "{}"
+    private_key:
+      filename: "{}"
+)EOF";
+
+      const std::string sds_content = fmt::format(
+          sds_template, server2_cert_rsa_,
+          TestEnvironment::runfilesPath("test/config/integration/certs/server2cert.pem"),
+          TestEnvironment::runfilesPath("test/config/integration/certs/server2key.pem"));
+
+      auto sds_path =
+          TestEnvironment::writeStringToFileForTest("server2_cert_rsa.sds.yaml", sds_content);
+      config_source->mutable_path_config_source()->set_path(sds_path);
+      config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3);
+    }
   }
 
   void createUpstreams() override {
@@ -317,6 +352,7 @@ version_info: "0"
 protected:
   Network::UpstreamTransportSocketFactoryPtr client_ssl_ctx_;
   bool dual_cert_{false};
+  bool multi_cert_{false};
 };
 
 INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, SdsDynamicDownstreamIntegrationTest,
@@ -476,6 +512,65 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, DualCert) {
   }
 }
 
+// A test that multiple(>2) sds configs are provided. More than one RSA/ECDSA cert
+// is allowed if they has different server name pattern(DNS SAN or subject name),
+// different server name patterns will be used to match SNI in cert selection.
+// Detailed behaviors of cert selection are tested in SslSocketTest with static
+// tls_certificates config. This test is to verify multiple certificates can be loaded
+// via sds and give a simple case of selecting cert based on SNI.
+TEST_P(SdsDynamicDownstreamIntegrationTest, MultipleCerts) {
+  on_server_init_function_ = [this]() {
+    createSdsStream(*(fake_upstreams_[1]));
+    sendSdsResponse(getServerSecretRsa());
+  };
+
+  multi_cert_ = true;
+  initialize();
+
+  client_ssl_ctx_ = createClientSslTransportSocketFactory(
+      ClientSslTransportOptions()
+          .setSni("www.lyft.com")
+          .setTlsVersion(envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLSv1_2)
+          .setCipherSuites({"ECDHE-RSA-AES128-GCM-SHA256"}),
+      context_manager_, *api_);
+  auto ssl_client1 = makeSslClientConnection();
+  codec_client_ = makeRawHttpConnection(std::move(ssl_client1), absl::nullopt);
+  EXPECT_TRUE(codec_client_->connected());
+  codec_client_->connection()->close(Network::ConnectionCloseType::NoFlush);
+  // peer certificate is not present when using QUIC
+  if (!test_quic_) {
+    EXPECT_EQ(TEST_SERVER_CERT_1_HASH,
+              codec_client_->connection()->ssl()->sha1PeerCertificateDigest());
+    EXPECT_EQ("www.lyft.com", codec_client_->connection()->ssl()->sni());
+  }
+
+  cleanupUpstreamAndDownstream();
+  client_ssl_ctx_ = createClientSslTransportSocketFactory(
+      ClientSslTransportOptions()
+          .setSni("www.lyft2.com")
+          .setTlsVersion(envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLSv1_2)
+          .setCipherSuites({"ECDHE-RSA-AES128-GCM-SHA256"}),
+      context_manager_, *api_);
+  auto ssl_client2 = makeSslClientConnection();
+  codec_client_ = makeRawHttpConnection(std::move(ssl_client2), absl::nullopt);
+  EXPECT_TRUE(codec_client_->connected());
+  codec_client_->connection()->close(Network::ConnectionCloseType::NoFlush);
+  // peer certificate is not present when using QUIC
+  if (!test_quic_) {
+    EXPECT_EQ(TEST_SERVER2_CERT_1_HASH,
+              codec_client_->connection()->ssl()->sha1PeerCertificateDigest());
+    EXPECT_EQ("www.lyft2.com", codec_client_->connection()->ssl()->sni());
+  }
+
+  // Success
+  EXPECT_EQ(1, test_server_->counter("sds.server_cert_rsa.update_success")->value());
+  EXPECT_EQ(0, test_server_->counter("sds.server_cert_rsa.update_rejected")->value());
+  EXPECT_EQ(1, test_server_->counter("sds.server2_cert_rsa.update_success")->value());
+  EXPECT_EQ(0, test_server_->counter("sds.server2_cert_rsa.update_rejected")->value());
+  EXPECT_EQ(1, test_server_->counter("sds.server_cert_ecdsa.update_success")->value());
+  EXPECT_EQ(0, test_server_->counter("sds.server_cert_ecdsa.update_rejected")->value());
+}
+
 // A test that SDS server send a bad secret for a static listener,
 // The first ssl request should fail at connecting.
 // then SDS send a good server secret,  the second request should be OK.