api: add 'redacted' option for protobuf messages, and redact SSL certs

api/envoy/api/v2/auth/cert.proto

@@ -11,6 +11,8 @@ import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/wrappers.proto";
 
+import "udpa/annotations/sensitive.proto";
+
 import "udpa/annotations/migrate.proto";
 import "validate/validate.proto";
 
@@ -128,7 +130,7 @@ message TlsCertificate {
   core.DataSource certificate_chain = 1;
 
   // The TLS private key.
-  core.DataSource private_key = 2;
+  core.DataSource private_key = 2 [(udpa.annotations.sensitive) = true];
 
   // BoringSSL private key method provider. This is an alternative to :ref:`private_key
   // <envoy_api_field_auth.TlsCertificate.private_key>` field. This can't be
@@ -141,7 +143,7 @@ message TlsCertificate {
 
   // The password to decrypt the TLS private key. If this field is not set, it is assumed that the
   // TLS private key is not password encrypted.
-  core.DataSource password = 3;
+  core.DataSource password = 3 [(udpa.annotations.sensitive) = true];
 
   // [#not-implemented-hide:]
   core.DataSource ocsp_staple = 4;
@@ -174,7 +176,8 @@ message TlsSessionTicketKeys {
   //   * Keep the session ticket keys at least as secure as your TLS certificate private keys
   //   * Rotate session ticket keys at least daily, and preferably hourly
   //   * Always generate keys using a cryptographically-secure random data source
-  repeated core.DataSource keys = 1 [(validate.rules).repeated = {min_items: 1}];
+  repeated core.DataSource keys = 1
+      [(validate.rules).repeated = {min_items: 1}, (udpa.annotations.sensitive) = true];
 }
 
 // [#next-free-field: 10]

api/envoy/extensions/transport_sockets/tls/v3alpha/cert.proto

@@ -11,6 +11,7 @@ import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/wrappers.proto";
 
+import "udpa/annotations/sensitive.proto";
 import "udpa/annotations/versioning.proto";
 
 import "validate/validate.proto";
@@ -136,7 +137,7 @@ message TlsCertificate {
   config.core.v3alpha.DataSource certificate_chain = 1;
 
   // The TLS private key.
-  config.core.v3alpha.DataSource private_key = 2;
+  config.core.v3alpha.DataSource private_key = 2 [(udpa.annotations.sensitive) = true];
 
   // BoringSSL private key method provider. This is an alternative to :ref:`private_key
   // <envoy_api_field_extensions.transport_sockets.tls.v3alpha.TlsCertificate.private_key>` field.
@@ -150,7 +151,7 @@ message TlsCertificate {
 
   // The password to decrypt the TLS private key. If this field is not set, it is assumed that the
   // TLS private key is not password encrypted.
-  config.core.v3alpha.DataSource password = 3;
+  config.core.v3alpha.DataSource password = 3 [(udpa.annotations.sensitive) = true];
 
   // [#not-implemented-hide:]
   config.core.v3alpha.DataSource ocsp_staple = 4;
@@ -187,7 +188,8 @@ message TlsSessionTicketKeys {
   //   * Keep the session ticket keys at least as secure as your TLS certificate private keys
   //   * Rotate session ticket keys at least daily, and preferably hourly
   //   * Always generate keys using a cryptographically-secure random data source
-  repeated config.core.v3alpha.DataSource keys = 1 [(validate.rules).repeated = {min_items: 1}];
+  repeated config.core.v3alpha.DataSource keys = 1
+      [(validate.rules).repeated = {min_items: 1}, (udpa.annotations.sensitive) = true];
 }
 
 // [#next-free-field: 10]

docs/root/operations/admin.rst

@@ -125,6 +125,14 @@ modify different aspects of the server:
   messages. See the :ref:`response definition <envoy_api_msg_admin.v2alpha.ConfigDump>` for more
   information.
 
+.. warning::
+  Configuration may include :ref:`TLS certificates <envoy_api_msg_auth.TlsCertificate>`. Before
+  dumping the configuration, Envoy will attempt to redact the ``private_key`` and ``password``
+  fields from any certificates it finds. This relies on the configuration being a strongly-typed
+  protobuf message. If your Envoy configuration uses deprecated ``config`` fields (of type
+  ``google.protobuf.Struct``), please update to the recommended ``typed_config`` fields (of type
+  ``google.protobuf.Any``) to ensure sensitive data is redacted properly.
+
 .. warning::
   The underlying proto is marked v2alpha and hence its contents, including the JSON representation,
   are not guaranteed to be stable.

source/common/protobuf/BUILD

@@ -64,9 +64,12 @@ envoy_cc_library(
         "//include/envoy/runtime:runtime_interface",
         "//source/common/common:assert_lib",
         "//source/common/common:hash_lib",
+        "//source/common/common:macros",
         "//source/common/common:utility_lib",
         "//source/common/config:api_type_oracle_lib",
         "//source/common/config:version_converter_lib",
+        "@com_github_cncf_udpa//udpa/annotations:pkg_cc_proto",
+        "@com_github_cncf_udpa//udpa/type/v1:pkg_cc_proto",
         "@envoy_api//envoy/type/v3alpha:pkg_cc_proto",
     ],
 )

source/common/protobuf/utility.cc

@@ -8,12 +8,15 @@
 
 #include "common/common/assert.h"
 #include "common/common/fmt.h"
+#include "common/common/macros.h"
 #include "common/config/api_type_oracle.h"
 #include "common/config/version_converter.h"
 #include "common/protobuf/message_validator_impl.h"
 #include "common/protobuf/protobuf.h"
 
 #include "absl/strings/match.h"
+#include "udpa/annotations/sensitive.pb.h"
+#include "udpa/type/v1/typed_struct.pb.h"
 #include "yaml-cpp/yaml.h"
 
 namespace Envoy {
@@ -596,6 +599,118 @@ std::string MessageUtil::CodeEnumToString(ProtobufUtil::error::Code code) {
   }
 }
 
+namespace {
+
+std::unique_ptr<Protobuf::Message> typeUrlToNewMessage(Protobuf::MessageFactory& message_factory,
+                                                       absl::string_view type_url) {
+  const absl::string_view type_name = TypeUtil::typeUrlToDescriptorFullName(type_url);
+  const Protobuf::Descriptor* descriptor =
+      Protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName(
+          static_cast<std::string>(type_name));
+  return std::unique_ptr<Protobuf::Message>(
+      descriptor == nullptr ? nullptr : message_factory.GetPrototype(descriptor)->New());
+}
+
+// Forward declaration for mutually-recursive helper functions.
+void redact(Protobuf::Message* message, bool ancestor_is_sensitive);
+
+// To redact an `Any`, we have to unpack it to its original type to redact it.
+bool redactAny(Protobuf::Message* message, bool ancestor_is_sensitive) {
+  auto* any = dynamic_cast<ProtobufWkt::Any*>(message);
+  if (any != nullptr) {
+    Protobuf::DynamicMessageFactory message_factory;
+    auto typed_message = typeUrlToNewMessage(message_factory, any->type_url());
+    if (typed_message == nullptr) {
+      // If the type URL doesn't correspond to a known proto, give up redacting and treat this
+      // message the same as a `ProtobufWkt::Struct`. See the documented limitation on
+      // `MessageUtil::redact()` for more context.
+      ENVOY_LOG_MISC(warn, "Could not redact ProtobufWkt::Any with unknown type URL {}",
+                     any->type_url());
+      return false;
+    }
+
+    any->UnpackTo(typed_message.get());
+    redact(typed_message.get(), ancestor_is_sensitive);
+    any->PackFrom(*typed_message);
+    return true;
+  }
+  return false;
+}
+
+// To redact a `TypedStruct`, we have to reify it based on its `type_url` to redact it.
+bool redactTypedStruct(Protobuf::Message* message, bool ancestor_is_sensitive) {
+  auto* typed_struct = dynamic_cast<udpa::type::v1::TypedStruct*>(message);
+  if (typed_struct != nullptr) {
+    Protobuf::DynamicMessageFactory message_factory;
+    auto typed_message = typeUrlToNewMessage(message_factory, typed_struct->type_url());
+    if (typed_message == nullptr) {
+      // If the type URL doesn't correspond to a known proto, give up redacting and treat this
+      // message the same as a `ProtobufWkt::Struct`. See the documented limitation on
+      // `MessageUtil::redact()` for more context.
+      ENVOY_LOG_MISC(warn, "Could not redact udpa::type::v1::TypedStruct with unknown type URL {}",
+                     typed_struct->type_url());
+      return false;
+    }
+
+    MessageUtil::jsonConvert(typed_struct->value(), ProtobufMessage::getNullValidationVisitor(),
+                             *typed_message);
+    redact(typed_message.get(), ancestor_is_sensitive);
+    MessageUtil::jsonConvert(*typed_message, *(typed_struct->mutable_value()));
+    return true;
+  }
+  return false;
+}
+
+// Recursive helper method for MessageUtil::redact() below.
+void redact(Protobuf::Message* message, bool ancestor_is_sensitive) {
+  if (redactAny(message, ancestor_is_sensitive) ||
+      redactTypedStruct(message, ancestor_is_sensitive)) {
+    return;
+  }
+
+  const auto* reflection = message->GetReflection();
+  std::vector<const Protobuf::FieldDescriptor*> field_descriptors;
+  reflection->ListFields(*message, &field_descriptors);
+  for (const auto* field_descriptor : field_descriptors) {
+    // Redact if this field or any of its ancestors have the `sensitive` option set.
+    const bool sensitive = ancestor_is_sensitive ||
+                           field_descriptor->options().GetExtension(udpa::annotations::sensitive);
+
+    if (field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_MESSAGE) {
+      // Recursive case: traverse message fields.
+      if (field_descriptor->is_repeated()) {
+        const int field_size = reflection->FieldSize(*message, field_descriptor);
+        for (int i = 0; i < field_size; ++i) {
+          redact(reflection->MutableRepeatedMessage(message, field_descriptor, i), sensitive);
+        }
+      } else {
+        redact(reflection->MutableMessage(message, field_descriptor), sensitive);
+      }
+    } else if (sensitive) {
+      // Base case: replace strings and bytes with "[redacted]" and clear all others.
+      if (field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_STRING ||
+          field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_BYTES) {
+        if (field_descriptor->is_repeated()) {
+          const int field_size = reflection->FieldSize(*message, field_descriptor);
+          for (int i = 0; i < field_size; ++i) {
+            reflection->SetRepeatedString(message, field_descriptor, i, "[redacted]");
+          }
+        } else {
+          reflection->SetString(message, field_descriptor, "[redacted]");
+        }
+      } else {
+        reflection->ClearField(message, field_descriptor);
+      }
+    }
+  }
+}
+
+} // namespace
+
+void MessageUtil::redact(Protobuf::Message& message) {
+  ::Envoy::redact(&message, /* ancestor_is_sensitive = */ false);
+}
+
 ProtobufWkt::Value ValueUtil::loadFromYaml(const std::string& yaml) {
   try {
     return parseYamlNode(YAML::Load(yaml));

source/common/protobuf/utility.h

@@ -362,6 +362,27 @@ class MessageUtil {
    * @param code the protobuf error code
    */
   static std::string CodeEnumToString(ProtobufUtil::error::Code code);
+
+  /**
+   * Modifies a message such that all sensitive data (that is, fields annotated as
+   * `udpa.annotations.sensitive`) is redacted for display. String-typed fields annotated as
+   * `sensitive` will be replaced with the string "[redacted]", bytes-typed fields will be replaced
+   * with the bytes `5B72656461637465645D` (the ASCII / UTF-8 encoding of the string "[redacted]"),
+   * primitive-typed fields (including enums) will be cleared, and message-typed fields will be
+   * traversed recursively to redact their contents.
+   *
+   * LIMITATION: This works properly for strongly-typed messages, as well as for messages packed in
+   * a `ProtobufWkt::Any` with a `type_url` corresponding to a proto that was compiled into the
+   * Envoy binary. However it does not work for messages encoded as `ProtobufWkt::Struct`, since
+   * structs are missing the "sensitive" annotations that this function expects. Similarly, it fails
+   * for messages encoded as `ProtobufWkt::Any` with a `type_url` that isn't registered with the
+   * binary. If you're working with struct-typed messages, including those that might be hiding
+   * within strongly-typed messages, please reify them to strongly-typed messages using
+   * `MessageUtil::jsonConvert()` before calling `MessageUtil::redact()`.
+   *
+   * @param message message to redact.
+   */
+  static void redact(Protobuf::Message& message);
 };
 
 class ValueUtil {

source/common/secret/BUILD

@@ -19,6 +19,7 @@ envoy_cc_library(
         "//include/envoy/server:transport_socket_config_interface",
         "//source/common/common:assert_lib",
         "//source/common/common:minimal_logger_lib",
+        "//source/common/protobuf:utility_lib",
         "@envoy_api//envoy/admin/v3alpha:pkg_cc_proto",
         "@envoy_api//envoy/config/core/v3alpha:pkg_cc_proto",
         "@envoy_api//envoy/extensions/transport_sockets/tls/v3alpha:pkg_cc_proto",