tls: support async cert validation (envoyproxy#21417)

Commit Message: change TLS transport socket to use SSL_CTX_set_custom_verify() instead of SSL_CTX_set_verify() and change CertValidator interface to support async cert validation. Also change EnvoyQuicCertVerifier to use the new async interfaces.

This change is needed for envoyproxy/envoy-mobile#1575. Envoy Mobile allows certificates to be verified by the OS-provided certificate verifier. And these verification can be very slow and so blocking the network thread while the verification happens is not an option. Instead, the verification should be performed asynchronously on a different thread. (This is how the cert verification works in Chrome, which is what we're modeling this implementation on).

Risk Level: high, change boring SSL interface used
Testing: added new unit tests and integration tests
Docs Changes: release note
Release Notes: documented tls transport changes.
Runtime guard: envoy.reloadable_features.tls_async_cert_validation

Signed-off-by: Dan Zhang <[email protected]>

Author: danzh2010

.clang-format

@@ -5,6 +5,7 @@ ColumnLimit: 100
 DerivePointerAlignment: false
 PointerAlignment: Left
 SortIncludes: false
+TypenameMacros: ['STACK_OF']
 ...
 
 ---

changelogs/current.yaml

@@ -9,6 +9,9 @@ behavior_changes:
   change: |
     Envoy no longer adds ``content-length: 0`` header when proxying UPGRADE requests without ``content-length`` and ``transfer-encoding`` headers.
     This behavior change can be reverted by setting the ``envoy.reloadable_features.http_skip_adding_content_length_to_upgrade`` runtime flag to false.
+- area: tls
+  change: |
+    Change TLS and QUIC transport sockets to support asynchronous cert validation extension. This behavior change can be reverted by setting runtime guard ``envoy.reloadable_features.tls_async_cert_validation`` to false.
 
 minor_behavior_changes:
 # *Changes that may cause incompatibilities for some users, but should not for most*

envoy/ssl/handshaker.h

@@ -38,6 +38,12 @@ class HandshakeCallbacks {
    * unset.
    */
   virtual Network::TransportSocketCallbacks* transportSocketCallbacks() PURE;
+
+  /**
+   * A callback to be called upon certificate validation completion if the validation is
+   * asynchronous.
+   */
+  virtual void onAsynchronousCertValidationComplete() PURE;
 };
 
 /**

envoy/ssl/ssl_socket_extended_info.h

@@ -1,16 +1,46 @@
 #pragma once
 
+#include <cstdint>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "envoy/common/pure.h"
+#include "envoy/event/dispatcher.h"
 
 namespace Envoy {
 namespace Ssl {
 
 enum class ClientValidationStatus { NotValidated, NoClientCertificate, Validated, Failed };
 
+enum class ValidateStatus {
+  NotStarted,
+  Pending,
+  Successful,
+  Failed,
+};
+
+/**
+ * Used to return the result from an asynchronous cert validation.
+ */
+class ValidateResultCallback {
+public:
+  virtual ~ValidateResultCallback() = default;
+
+  virtual Event::Dispatcher& dispatcher() PURE;
+
+  /**
+   * Called when the asynchronous cert validation completes.
+   * @param succeeded true if the validation succeeds
+   * @param error_details failure details, only used if the validation fails.
+   * @param tls_alert the TLS error related to the failure, only used if the validation fails.
+   */
+  virtual void onCertValidationResult(bool succeeded, const std::string& error_details,
+                                      uint8_t tls_alert) PURE;
+};
+
+using ValidateResultCallbackPtr = std::unique_ptr<ValidateResultCallback>;
+
 class SslExtendedSocketInfo {
 public:
   virtual ~SslExtendedSocketInfo() = default;
@@ -24,6 +54,30 @@ class SslExtendedSocketInfo {
    * @return ClientValidationStatus The peer certificate validation status.
    **/
   virtual ClientValidationStatus certificateValidationStatus() const PURE;
+
+  /**
+   * Only called when doing asynchronous cert validation.
+   * @return ValidateResultCallbackPtr a callback used to return the validation result.
+   */
+  virtual ValidateResultCallbackPtr createValidateResultCallback() PURE;
+
+  /**
+   * Called after the cert validation completes either synchronously or asynchronously.
+   * @param succeeded true if the validation succeeded.
+   */
+  virtual void onCertificateValidationCompleted(bool succeeded) PURE;
+
+  /**
+   * @return ValidateStatus the validation status.
+   */
+  virtual ValidateStatus certificateValidationResult() const PURE;
+
+  /**
+   * Called when doing asynchronous cert validation.
+   * @return uint8_t represents the TLS alert populated by cert validator in
+   * case of failure.
+   */
+  virtual uint8_t certificateValidationAlert() const PURE;
 };
 
 } // namespace Ssl

source/common/quic/BUILD

@@ -131,6 +131,7 @@ envoy_cc_library(
     deps = [
         ":envoy_quic_proof_verifier_base_lib",
         ":envoy_quic_utils_lib",
+        ":quic_ssl_connection_info_lib",
         "//source/extensions/transport_sockets/tls:context_lib",
     ],
 )

source/common/quic/envoy_quic_client_session.cc

@@ -1,5 +1,9 @@
 #include "source/common/quic/envoy_quic_client_session.h"
 
+#include <openssl/ssl.h>
+
+#include <memory>
+
 #include "source/common/event/dispatcher_impl.h"
 #include "source/common/quic/envoy_quic_proof_verifier.h"
 #include "source/common/quic/envoy_quic_utils.h"
@@ -14,9 +18,10 @@ class EnvoyQuicProofVerifyContextImpl : public EnvoyQuicProofVerifyContext {
 public:
   EnvoyQuicProofVerifyContextImpl(
       Event::Dispatcher& dispatcher, const bool is_server,
-      const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options)
+      const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
+      QuicSslConnectionInfo& ssl_info)
       : dispatcher_(dispatcher), is_server_(is_server),
-        transport_socket_options_(transport_socket_options) {}
+        transport_socket_options_(transport_socket_options), ssl_info_(ssl_info) {}
 
   // EnvoyQuicProofVerifyContext
   bool isServer() const override { return is_server_; }
@@ -25,10 +30,17 @@ class EnvoyQuicProofVerifyContextImpl : public EnvoyQuicProofVerifyContext {
     return transport_socket_options_;
   }
 
+  Extensions::TransportSockets::Tls::CertValidator::ExtraValidationContext
+  extraValidationContext() const override {
+    ASSERT(ssl_info_.ssl());
+    return {};
+  }
+
 private:
   Event::Dispatcher& dispatcher_;
   const bool is_server_;
   const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options_;
+  QuicSslConnectionInfo& ssl_info_;
 };
 
 EnvoyQuicClientSession::EnvoyQuicClientSession(
@@ -183,7 +195,7 @@ std::unique_ptr<quic::QuicCryptoClientStreamBase> EnvoyQuicClientSession::Create
   return crypto_stream_factory_.createEnvoyQuicCryptoClientStream(
       server_id(), this,
       std::make_unique<EnvoyQuicProofVerifyContextImpl>(dispatcher_, /*is_server=*/false,
-                                                        transport_socket_options_),
+                                                        transport_socket_options_, *quic_ssl_info_),
       crypto_config(), this, /*has_application_state = */ version().UsesHttp3());
 }
 

source/common/quic/envoy_quic_proof_verifier.cc

@@ -1,13 +1,73 @@
 #include "source/common/quic/envoy_quic_proof_verifier.h"
 
+#include <openssl/ssl.h>
+
+#include <cstdint>
+#include <memory>
+
 #include "source/common/quic/envoy_quic_utils.h"
+#include "source/common/runtime/runtime_features.h"
 #include "source/extensions/transport_sockets/tls/utility.h"
 
 #include "quiche/quic/core/crypto/certificate_view.h"
 
 namespace Envoy {
 namespace Quic {
 
+using ValidationResults = Envoy::Extensions::TransportSockets::Tls::ValidationResults;
+
+namespace {
+
+// Returns true if hostname matches one of the Subject Alt Names in cert_view. Returns false and
+// sets error_details otherwise
+bool verifyLeafCertMatchesHostname(quic::CertificateView& cert_view, const std::string& hostname,
+                                   std::string* error_details) {
+  for (const absl::string_view& config_san : cert_view.subject_alt_name_domains()) {
+    if (Extensions::TransportSockets::Tls::Utility::dnsNameMatch(hostname, config_san)) {
+      return true;
+    }
+  }
+  *error_details = absl::StrCat("Leaf certificate doesn't match hostname: ", hostname);
+  return false;
+}
+
+class QuicValidateResultCallback : public Ssl::ValidateResultCallback {
+public:
+  QuicValidateResultCallback(Event::Dispatcher& dispatcher,
+                             std::unique_ptr<quic::ProofVerifierCallback>&& quic_callback,
+                             const std::string& hostname, const std::string& leaf_cert)
+      : dispatcher_(dispatcher), quic_callback_(std::move(quic_callback)), hostname_(hostname),
+        leaf_cert_(leaf_cert) {}
+
+  Event::Dispatcher& dispatcher() override { return dispatcher_; }
+
+  void onCertValidationResult(bool succeeded, const std::string& error_details,
+                              uint8_t /*tls_alert*/) override {
+    if (!succeeded) {
+      std::unique_ptr<quic::ProofVerifyDetails> details = std::make_unique<CertVerifyResult>(false);
+      quic_callback_->Run(succeeded, error_details, &details);
+      return;
+    }
+    std::string error;
+
+    std::unique_ptr<quic::CertificateView> cert_view =
+        quic::CertificateView::ParseSingleCertificate(leaf_cert_);
+    succeeded = verifyLeafCertMatchesHostname(*cert_view, hostname_, &error);
+    std::unique_ptr<quic::ProofVerifyDetails> details =
+        std::make_unique<CertVerifyResult>(succeeded);
+    quic_callback_->Run(succeeded, error, &details);
+  }
+
+private:
+  Event::Dispatcher& dispatcher_;
+  std::unique_ptr<quic::ProofVerifierCallback> quic_callback_;
+  const std::string hostname_;
+  // Leaf cert needs to be retained in case of asynchronous validation.
+  std::string leaf_cert_;
+};
+
+} // namespace
+
 quic::QuicAsyncStatus EnvoyQuicProofVerifier::VerifyCertChain(
     const std::string& hostname, const uint16_t port, const std::vector<std::string>& certs,
     const std::string& ocsp_response, const std::string& cert_sct,
@@ -23,11 +83,61 @@ quic::QuicAsyncStatus EnvoyQuicProofVerifier::VerifyCertChain(
   }
   ENVOY_BUG(!verify_context->isServer(), "Client certificates are not supported in QUIC yet.");
 
-  if (doVerifyCertChain(hostname, port, certs, ocsp_response, cert_sct, context, error_details,
-                        out_alert, std::move(callback))) {
-    *details = std::make_unique<CertVerifyResult>(true);
-    return quic::QUIC_SUCCESS;
+  if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.tls_async_cert_validation")) {
+    if (doVerifyCertChain(hostname, port, certs, ocsp_response, cert_sct, context, error_details,
+                          out_alert, std::move(callback))) {
+      *details = std::make_unique<CertVerifyResult>(true);
+      return quic::QUIC_SUCCESS;
+    }
+    *details = std::make_unique<CertVerifyResult>(false);
+    return quic::QUIC_FAILURE;
   }
+
+  bssl::UniquePtr<STACK_OF(X509)> cert_chain(sk_X509_new_null());
+  for (const auto& cert_str : certs) {
+    bssl::UniquePtr<X509> cert = parseDERCertificate(cert_str, error_details);
+    if (!cert || !bssl::PushToStack(cert_chain.get(), std::move(cert))) {
+      return quic::QUIC_FAILURE;
+    }
+  }
+  std::unique_ptr<quic::CertificateView> cert_view =
+      quic::CertificateView::ParseSingleCertificate(certs[0]);
+  ASSERT(cert_view != nullptr);
+  int sign_alg = deduceSignatureAlgorithmFromPublicKey(cert_view->public_key(), error_details);
+  if (sign_alg == 0) {
+    return quic::QUIC_FAILURE;
+  }
+
+  auto envoy_callback = std::make_unique<QuicValidateResultCallback>(
+      verify_context->dispatcher(), std::move(callback), hostname, certs[0]);
+  ASSERT(dynamic_cast<Extensions::TransportSockets::Tls::ClientContextImpl*>(context_.get()) !=
+         nullptr);
+  // We down cast rather than add customVerifyCertChainForQuic to Envoy::Ssl::Context because
+  // verifyCertChain uses a bunch of SSL-specific structs which we want to keep out of the interface
+  // definition.
+  ValidationResults result =
+      static_cast<Extensions::TransportSockets::Tls::ClientContextImpl*>(context_.get())
+          ->customVerifyCertChainForQuic(
+              *cert_chain, std::move(envoy_callback), verify_context->isServer(),
+              verify_context->transportSocketOptions(), verify_context->extraValidationContext());
+  if (result.status == ValidationResults::ValidationStatus::Pending) {
+    return quic::QUIC_PENDING;
+  }
+  if (result.status == ValidationResults::ValidationStatus::Successful) {
+    if (verifyLeafCertMatchesHostname(*cert_view, hostname, error_details)) {
+      *details = std::make_unique<CertVerifyResult>(true);
+      return quic::QUIC_SUCCESS;
+    }
+  } else {
+    ASSERT(result.status == ValidationResults::ValidationStatus::Failed);
+    if (result.error_details.has_value() && error_details) {
+      *error_details = std::move(result.error_details.value());
+    }
+    if (result.tls_alert.has_value() && out_alert) {
+      *out_alert = result.tls_alert.value();
+    }
+  }
+
   *details = std::make_unique<CertVerifyResult>(false);
   return quic::QUIC_FAILURE;
 }
@@ -65,11 +175,8 @@ bool EnvoyQuicProofVerifier::doVerifyCertChain(
   if (!success) {
     return false;
   }
-
-  for (const absl::string_view& config_san : cert_view->subject_alt_name_domains()) {
-    if (Extensions::TransportSockets::Tls::Utility::dnsNameMatch(hostname, config_san)) {
-      return true;
-    }
+  if (verifyLeafCertMatchesHostname(*cert_view, hostname, error_details)) {
+    return true;
   }
   *error_details = absl::StrCat("Leaf certificate doesn't match hostname: ", hostname);
   return false;

source/common/quic/envoy_quic_proof_verifier.h

@@ -1,6 +1,9 @@
 #pragma once
 
+#include <memory>
+
 #include "source/common/quic/envoy_quic_proof_verifier_base.h"
+#include "source/common/quic/quic_ssl_connection_info.h"
 #include "source/extensions/transport_sockets/tls/context_impl.h"
 
 namespace Envoy {
@@ -18,19 +21,25 @@ class CertVerifyResult : public quic::ProofVerifyDetails {
   bool is_valid_{false};
 };
 
+using CertVerifyResultPtr = std::unique_ptr<CertVerifyResult>();
+
 // An interface for the Envoy specific QUIC verify context.
 class EnvoyQuicProofVerifyContext : public quic::ProofVerifyContext {
 public:
   virtual Event::Dispatcher& dispatcher() const PURE;
   virtual bool isServer() const PURE;
   virtual const Network::TransportSocketOptionsConstSharedPtr& transportSocketOptions() const PURE;
+  virtual Extensions::TransportSockets::Tls::CertValidator::ExtraValidationContext
+  extraValidationContext() const PURE;
 };
 
+using EnvoyQuicProofVerifyContextPtr = std::unique_ptr<EnvoyQuicProofVerifyContext>;
+
 // A quic::ProofVerifier implementation which verifies cert chain using SSL
 // client context config.
 class EnvoyQuicProofVerifier : public EnvoyQuicProofVerifierBase {
 public:
-  EnvoyQuicProofVerifier(Envoy::Ssl::ClientContextSharedPtr&& context)
+  explicit EnvoyQuicProofVerifier(Envoy::Ssl::ClientContextSharedPtr&& context)
       : context_(std::move(context)) {
     ASSERT(context_.get());
   }
@@ -45,6 +54,7 @@ class EnvoyQuicProofVerifier : public EnvoyQuicProofVerifierBase {
                   std::unique_ptr<quic::ProofVerifierCallback> callback) override;
 
 private:
+  // TODO(danzh) remove when deprecating envoy.reloadable_features.tls_async_cert_validation.
   bool doVerifyCertChain(const std::string& hostname, const uint16_t port,
                          const std::vector<std::string>& certs, const std::string& ocsp_response,
                          const std::string& cert_sct, const quic::ProofVerifyContext* context,

source/common/runtime/runtime_features.cc

@@ -62,6 +62,7 @@ RUNTIME_GUARD(envoy_reloadable_features_skip_delay_close);
 RUNTIME_GUARD(envoy_reloadable_features_strict_check_on_ipv4_compat);
 RUNTIME_GUARD(envoy_reloadable_features_support_locality_update_on_eds_cluster_endpoints);
 RUNTIME_GUARD(envoy_reloadable_features_test_feature_true);
+RUNTIME_GUARD(envoy_reloadable_features_tls_async_cert_validation);
 RUNTIME_GUARD(envoy_reloadable_features_top_level_ecds_stats);
 RUNTIME_GUARD(envoy_reloadable_features_update_grpc_response_error_tag);
 RUNTIME_GUARD(envoy_reloadable_features_use_rfc_connect);