api: add 'redacted' option for protobuf messages, and redact SSL certs

api/envoy/api/v2/auth/cert.proto

@@ -11,6 +11,8 @@ import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/wrappers.proto";
 
+import "udpa/annotations/sensitive.proto";
+
 import "udpa/annotations/migrate.proto";
 import "validate/validate.proto";
 
@@ -128,7 +130,7 @@ message TlsCertificate {
   core.DataSource certificate_chain = 1;
 
   // The TLS private key.
-  core.DataSource private_key = 2;
+  core.DataSource private_key = 2 [(udpa.annotations.sensitive) = true];
 
   // BoringSSL private key method provider. This is an alternative to :ref:`private_key
   // <envoy_api_field_auth.TlsCertificate.private_key>` field. This can't be
@@ -141,7 +143,7 @@ message TlsCertificate {
 
   // The password to decrypt the TLS private key. If this field is not set, it is assumed that the
   // TLS private key is not password encrypted.
-  core.DataSource password = 3;
+  core.DataSource password = 3 [(udpa.annotations.sensitive) = true];
 
   // [#not-implemented-hide:]
   core.DataSource ocsp_staple = 4;
@@ -174,7 +176,8 @@ message TlsSessionTicketKeys {
   //   * Keep the session ticket keys at least as secure as your TLS certificate private keys
   //   * Rotate session ticket keys at least daily, and preferably hourly
   //   * Always generate keys using a cryptographically-secure random data source
-  repeated core.DataSource keys = 1 [(validate.rules).repeated = {min_items: 1}];
+  repeated core.DataSource keys = 1
+      [(validate.rules).repeated = {min_items: 1}, (udpa.annotations.sensitive) = true];
 }
 
 // [#next-free-field: 10]

api/envoy/extensions/transport_sockets/tls/v3alpha/cert.proto

@@ -11,6 +11,7 @@ import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/wrappers.proto";
 
+import "udpa/annotations/sensitive.proto";
 import "udpa/annotations/versioning.proto";
 
 import "validate/validate.proto";
@@ -136,7 +137,7 @@ message TlsCertificate {
   config.core.v3alpha.DataSource certificate_chain = 1;
 
   // The TLS private key.
-  config.core.v3alpha.DataSource private_key = 2;
+  config.core.v3alpha.DataSource private_key = 2 [(udpa.annotations.sensitive) = true];
 
   // BoringSSL private key method provider. This is an alternative to :ref:`private_key
   // <envoy_api_field_extensions.transport_sockets.tls.v3alpha.TlsCertificate.private_key>` field.
@@ -150,7 +151,7 @@ message TlsCertificate {
 
   // The password to decrypt the TLS private key. If this field is not set, it is assumed that the
   // TLS private key is not password encrypted.
-  config.core.v3alpha.DataSource password = 3;
+  config.core.v3alpha.DataSource password = 3 [(udpa.annotations.sensitive) = true];
 
   // [#not-implemented-hide:]
   config.core.v3alpha.DataSource ocsp_staple = 4;
@@ -187,7 +188,8 @@ message TlsSessionTicketKeys {
   //   * Keep the session ticket keys at least as secure as your TLS certificate private keys
   //   * Rotate session ticket keys at least daily, and preferably hourly
   //   * Always generate keys using a cryptographically-secure random data source
-  repeated config.core.v3alpha.DataSource keys = 1 [(validate.rules).repeated = {min_items: 1}];
+  repeated config.core.v3alpha.DataSource keys = 1
+      [(validate.rules).repeated = {min_items: 1}, (udpa.annotations.sensitive) = true];
 }
 
 // [#next-free-field: 10]

docs/root/operations/admin.rst

@@ -125,6 +125,14 @@ modify different aspects of the server:
   messages. See the :ref:`response definition <envoy_api_msg_admin.v2alpha.ConfigDump>` for more
   information.
 
+.. warning::
+  Configuration may include :ref:`TLS certificates <envoy_api_msg_auth.TlsCertificate>`. Before
+  dumping the configuration, Envoy will attempt to redact the ``private_key`` and ``password``
+  fields from any certificates it finds. This relies on the configuration being a strongly-typed
+  protobuf message. If your Envoy configuration uses deprecated ``config`` fields (of type
+  ``google.protobuf.Struct``), please update to the recommended ``typed_config`` fields (of type
+  ``google.protobuf.Any``) to ensure sensitive data is redacted properly.
+
 .. warning::
   The underlying proto is marked v2alpha and hence its contents, including the JSON representation,
   are not guaranteed to be stable.

source/common/protobuf/BUILD

@@ -68,6 +68,7 @@ envoy_cc_library(
         "//source/common/common:utility_lib",
         "//source/common/config:api_type_oracle_lib",
         "//source/common/config:version_converter_lib",
+        "@com_github_cncf_udpa//udpa/annotations:pkg_cc_proto",
         "@envoy_api//envoy/annotations:pkg_cc_proto",
         "@envoy_api//envoy/type/v3alpha:pkg_cc_proto",
     ],

source/common/protobuf/utility.cc

@@ -16,6 +16,7 @@
 #include "common/protobuf/well_known.h"
 
 #include "absl/strings/match.h"
+#include "udpa/annotations/sensitive.pb.h"
 #include "yaml-cpp/yaml.h"
 
 namespace Envoy {
@@ -613,6 +614,147 @@ std::string MessageUtil::CodeEnumToString(ProtobufUtil::error::Code code) {
   }
 }
 
+namespace {
+
+// Forward declaration for mutually-recursive helper functions.
+void redact(Protobuf::Message* message, bool ancestor_is_sensitive);
+
+using Transform = std::function<void(Protobuf::Message*, const Protobuf::Reflection*,
+                                     const Protobuf::FieldDescriptor*)>;
+
+// To redact opaque types, namely `Any` and `TypedStruct`, we have to reify them to the concrete
+// message types specified by their `type_url` before we can redact their contents. This is mostly
+// identical between `Any` and `TypedStruct`, the only difference being how they are packed and
+// unpacked. Note that we have to use reflection on the opaque type here, rather than downcasting
+// to `Any` or `TypedStruct`, because any message we might be handling could have originated from
+// a `DynamicMessageFactory`.
+bool redactOpaque(Protobuf::Message* message, bool ancestor_is_sensitive,
+                  absl::string_view opaque_type_name, Transform unpack, Transform repack) {
+  // Ensure this message has the opaque type we're expecting.
+  const auto* opaque_descriptor = message->GetDescriptor();
+  if (opaque_descriptor->full_name() != opaque_type_name) {
+    return false;
+  }
+
+  // Find descriptors for the `type_url` and `value` fields. The `type_url` field must not be
+  // empty, but `value` may be (in which case our work is done).
+  const auto* reflection = message->GetReflection();
+  const auto* type_url_field_descriptor = opaque_descriptor->FindFieldByName("type_url");
+  const auto* value_field_descriptor = opaque_descriptor->FindFieldByName("value");
+  ASSERT(type_url_field_descriptor != nullptr && value_field_descriptor != nullptr &&
+         reflection->HasField(*message, type_url_field_descriptor));
+  if (!reflection->HasField(*message, value_field_descriptor)) {
+    return true;
+  }
+
+  // Try to find a descriptor for `type_url` in the pool and instantiate a new message of the
+  // correct concrete type.
+  const std::string type_url(reflection->GetString(*message, type_url_field_descriptor));
+  const std::string concrete_type_name(TypeUtil::typeUrlToDescriptorFullName(type_url));
+  const auto* concrete_descriptor =
+      Protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName(concrete_type_name);
+  if (concrete_descriptor == nullptr) {
+    // If the type URL doesn't correspond to a known proto, don't try to reify it, just treat it
+    // like any other message. See the documented limitation on `MessageUtil::redact()` for more
+    // context.
+    ENVOY_LOG_MISC(warn, "Could not reify {} with unknown type URL {}", opaque_type_name, type_url);
+    return false;
+  }
+  Protobuf::DynamicMessageFactory message_factory;
+  std::unique_ptr<Protobuf::Message> typed_message(
+      message_factory.GetPrototype(concrete_descriptor)->New());
+
+  // Finally we can unpack, redact, and repack the opaque message using the provided callbacks.
+  unpack(typed_message.get(), reflection, value_field_descriptor);
+  redact(typed_message.get(), ancestor_is_sensitive);
+  repack(typed_message.get(), reflection, value_field_descriptor);
+  return true;
+}
+
+bool redactAny(Protobuf::Message* message, bool ancestor_is_sensitive) {
+  return redactOpaque(
+      message, ancestor_is_sensitive, "google.protobuf.Any",
+      [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,
+                const Protobuf::FieldDescriptor* field_descriptor) {
+        // To unpack an `Any`, parse the serialized proto.
+        typed_message->ParseFromString(reflection->GetString(*message, field_descriptor));
+      },
+      [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,
+                const Protobuf::FieldDescriptor* field_descriptor) {
+        // To repack an `Any`, reserialize its proto.
+        reflection->SetString(message, field_descriptor, typed_message->SerializeAsString());
+      });
+}
+
+// To redact a `TypedStruct`, we have to reify it based on its `type_url` to redact it.
+bool redactTypedStruct(Protobuf::Message* message, bool ancestor_is_sensitive) {
+  return redactOpaque(
+      message, ancestor_is_sensitive, "udpa.type.v1.TypedStruct",
+      [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,
+                const Protobuf::FieldDescriptor* field_descriptor) {
+        // To unpack a `TypedStruct`, convert the struct from JSON.
+        jsonConvertInternal(reflection->GetMessage(*message, field_descriptor),
+                            ProtobufMessage::getNullValidationVisitor(), *typed_message);
+      },
+      [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,
+                const Protobuf::FieldDescriptor* field_descriptor) {
+        // To repack a `TypedStruct`, convert the message back to JSON.
+        jsonConvertInternal(*typed_message, ProtobufMessage::getNullValidationVisitor(),
+                            *(reflection->MutableMessage(message, field_descriptor)));
+      });
+}
+
+// Recursive helper method for MessageUtil::redact() below.
+void redact(Protobuf::Message* message, bool ancestor_is_sensitive) {
+  if (redactAny(message, ancestor_is_sensitive) ||
+      redactTypedStruct(message, ancestor_is_sensitive)) {
+    return;
+  }
+
+  const auto* descriptor = message->GetDescriptor();
+  const auto* reflection = message->GetReflection();
+  for (int i = 0; i < descriptor->field_count(); ++i) {
+    const auto* field_descriptor = descriptor->field(i);
+
+    // Redact if this field or any of its ancestors have the `sensitive` option set.
+    const bool sensitive = ancestor_is_sensitive ||
+                           field_descriptor->options().GetExtension(udpa::annotations::sensitive);
+
+    if (field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_MESSAGE) {
+      // Recursive case: traverse message fields.
+      if (field_descriptor->is_repeated()) {
+        const int field_size = reflection->FieldSize(*message, field_descriptor);
+        for (int i = 0; i < field_size; ++i) {
+          redact(reflection->MutableRepeatedMessage(message, field_descriptor, i), sensitive);
+        }
+      } else if (reflection->HasField(*message, field_descriptor)) {
+        redact(reflection->MutableMessage(message, field_descriptor), sensitive);
+      }
+    } else if (sensitive) {
+      // Base case: replace strings and bytes with "[redacted]" and clear all others.
+      if (field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_STRING ||
+          field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_BYTES) {
+        if (field_descriptor->is_repeated()) {
+          const int field_size = reflection->FieldSize(*message, field_descriptor);
+          for (int i = 0; i < field_size; ++i) {
+            reflection->SetRepeatedString(message, field_descriptor, i, "[redacted]");
+          }
+        } else if (reflection->HasField(*message, field_descriptor)) {
+          reflection->SetString(message, field_descriptor, "[redacted]");
+        }
+      } else {
+        reflection->ClearField(message, field_descriptor);
+      }
+    }
+  }
+}
+
+} // namespace
+
+void MessageUtil::redact(Protobuf::Message& message) {
+  ::Envoy::redact(&message, /* ancestor_is_sensitive = */ false);
+}
+
 ProtobufWkt::Value ValueUtil::loadFromYaml(const std::string& yaml) {
   try {
     return parseYamlNode(YAML::Load(yaml));

source/common/protobuf/utility.h

@@ -363,6 +363,27 @@ class MessageUtil {
    * @param code the protobuf error code
    */
   static std::string CodeEnumToString(ProtobufUtil::error::Code code);
+
+  /**
+   * Modifies a message such that all sensitive data (that is, fields annotated as
+   * `udpa.annotations.sensitive`) is redacted for display. String-typed fields annotated as
+   * `sensitive` will be replaced with the string "[redacted]", bytes-typed fields will be replaced
+   * with the bytes `5B72656461637465645D` (the ASCII / UTF-8 encoding of the string "[redacted]"),
+   * primitive-typed fields (including enums) will be cleared, and message-typed fields will be
+   * traversed recursively to redact their contents.
+   *
+   * LIMITATION: This works properly for strongly-typed messages, as well as for messages packed in
+   * a `ProtobufWkt::Any` with a `type_url` corresponding to a proto that was compiled into the
+   * Envoy binary. However it does not work for messages encoded as `ProtobufWkt::Struct`, since
+   * structs are missing the "sensitive" annotations that this function expects. Similarly, it fails
+   * for messages encoded as `ProtobufWkt::Any` with a `type_url` that isn't registered with the
+   * binary. If you're working with struct-typed messages, including those that might be hiding
+   * within strongly-typed messages, please reify them to strongly-typed messages using
+   * `MessageUtil::jsonConvert()` before calling `MessageUtil::redact()`.
+   *
+   * @param message message to redact.
+   */
+  static void redact(Protobuf::Message& message);
 };
 
 class ValueUtil {

source/common/secret/BUILD

@@ -20,6 +20,7 @@ envoy_cc_library(
         "//source/common/common:assert_lib",
         "//source/common/common:minimal_logger_lib",
         "//source/common/config:version_converter_lib",
+        "//source/common/protobuf:utility_lib",
         "@envoy_api//envoy/admin/v3alpha:pkg_cc_proto",
         "@envoy_api//envoy/config/core/v3alpha:pkg_cc_proto",
         "@envoy_api//envoy/extensions/transport_sockets/tls/v3alpha:pkg_cc_proto",