api: add 'redacted' option for protobuf messages, and redact SSL certs

api/bazel/repository_locations.bzl

@@ -15,8 +15,8 @@ PROMETHEUS_SHA = "783bdaf8ee0464b35ec0c8704871e1e72afa0005c3f3587f65d9d6694bf391
 
 KAFKA_SOURCE_SHA = "ae7a1696c0a0302b43c5b21e515c37e6ecd365941f68a510a7e442eebddf39a1"  # 2.2.0-rc2
 
-UDPA_GIT_SHA = "d1f2ba7f5ba62c55b7466409e7f972c93e957d2b"  # Dec 6, 2019
-UDPA_SHA256 = "0271fb8ad2ec9ade21e4c7737dd128d2a8d8edebe911b777e2fc2585414aa045"
+UDPA_GIT_SHA = "89624d5efc546eac78cf2784bed4dc36a09d5772"  # Dec 12, 2019
+UDPA_SHA256 = "4ec5716adc13459bb23ee11b2d842fb4da7ae10add113f4e2d9eed1659c34b44"
 
 ZIPKINAPI_RELEASE = "0.2.2"  # Aug 23, 2019
 ZIPKINAPI_SHA256 = "688c4fe170821dd589f36ec45aaadc03a618a40283bc1f97da8fa11686fc816b"

api/envoy/api/v2/auth/BUILD

@@ -5,5 +5,8 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
 licenses(["notice"])  # Apache 2
 
 api_proto_package(
-    deps = ["//envoy/api/v2/core:pkg"],
+    deps = [
+        "//envoy/api/v2/core:pkg",
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+    ],
 )

api/envoy/api/v2/auth/cert.proto

@@ -13,6 +13,8 @@ import "google/protobuf/any.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/wrappers.proto";
 
+import "udpa/annotations/sensitive.proto";
+
 import "validate/validate.proto";
 
 // [#protodoc-title: Common TLS configuration]
@@ -123,7 +125,7 @@ message TlsCertificate {
   core.DataSource certificate_chain = 1;
 
   // The TLS private key.
-  core.DataSource private_key = 2;
+  core.DataSource private_key = 2 [(udpa.annotations.sensitive) = true];
 
   // BoringSSL private key method provider. This is an alternative to :ref:`private_key
   // <envoy_api_field_auth.TlsCertificate.private_key>` field. This can't be
@@ -136,7 +138,7 @@ message TlsCertificate {
 
   // The password to decrypt the TLS private key. If this field is not set, it is assumed that the
   // TLS private key is not password encrypted.
-  core.DataSource password = 3;
+  core.DataSource password = 3 [(udpa.annotations.sensitive) = true];
 
   // [#not-implemented-hide:]
   core.DataSource ocsp_staple = 4;

api/envoy/api/v3alpha/auth/cert.proto

@@ -13,6 +13,7 @@ import "google/protobuf/any.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/wrappers.proto";
 
+import "udpa/annotations/sensitive.proto";
 import "udpa/annotations/versioning.proto";
 
 import "validate/validate.proto";
@@ -134,7 +135,7 @@ message TlsCertificate {
   core.DataSource certificate_chain = 1;
 
   // The TLS private key.
-  core.DataSource private_key = 2;
+  core.DataSource private_key = 2 [(udpa.annotations.sensitive) = true];
 
   // BoringSSL private key method provider. This is an alternative to :ref:`private_key
   // <envoy_api_field_api.v3alpha.auth.TlsCertificate.private_key>` field. This can't be
@@ -147,7 +148,7 @@ message TlsCertificate {
 
   // The password to decrypt the TLS private key. If this field is not set, it is assumed that the
   // TLS private key is not password encrypted.
-  core.DataSource password = 3;
+  core.DataSource password = 3 [(udpa.annotations.sensitive) = true];
 
   // [#not-implemented-hide:]
   core.DataSource ocsp_staple = 4;

docs/root/operations/admin.rst

@@ -125,6 +125,14 @@ modify different aspects of the server:
   messages. See the :ref:`response definition <envoy_api_msg_admin.v2alpha.ConfigDump>` for more
   information.
 
+.. warning::
+  Configuration may include :ref:`TLS certificates <envoy_api_msg_auth.TlsCertificate>`. Before
+  dumping the configuration, Envoy will attempt to redact the ``private_key`` and ``password``
+  fields from any certificates it finds. This relies on the configuration being a strongly-typed
+  protobuf message. If your Envoy configuration uses deprecated ``config`` fields (of type
+  ``google.protobuf.Struct``), please update to the recommended ``typed_config`` fields (of type
+  ``google.protobuf.Any``) to ensure sensitive data is redacted properly.
+
 .. warning::
   The underlying proto is marked v2alpha and hence its contents, including the JSON representation,
   are not guaranteed to be stable.

source/common/protobuf/BUILD

@@ -64,6 +64,8 @@ envoy_cc_library(
         "//source/common/common:assert_lib",
         "//source/common/common:hash_lib",
         "//source/common/common:utility_lib",
+        "@com_github_cncf_udpa//udpa/annotations:pkg_cc_proto",
+        "@com_github_cncf_udpa//udpa/type/v1:pkg_cc_proto",
         "@envoy_api//envoy/type:pkg_cc_proto",
     ],
 )

source/common/protobuf/utility.cc

@@ -12,6 +12,8 @@
 #include "common/protobuf/protobuf.h"
 
 #include "absl/strings/match.h"
+#include "udpa/annotations/sensitive.pb.h"
+#include "udpa/type/v1/typed_struct.pb.h"
 #include "yaml-cpp/yaml.h"
 
 namespace Envoy {
@@ -506,6 +508,114 @@ std::string MessageUtil::CodeEnumToString(ProtobufUtil::error::Code code) {
   }
 }
 
+namespace {
+
+std::unique_ptr<Protobuf::Message> typeUrlToNewMessage(Protobuf::MessageFactory& message_factory,
+                                                       absl::string_view type_url) {
+  const absl::string_view type_name = TypeUtil::typeUrlToDescriptorFullName(type_url);
+  const Protobuf::Descriptor* descriptor =
+      Protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName(
+          static_cast<std::string>(type_name));
+  return std::unique_ptr<Protobuf::Message>(
+      descriptor == nullptr ? nullptr : message_factory.GetPrototype(descriptor)->New());
+}
+
+// Recursive helper method for MessageUtil::redact() below. Note that we have to keep track of
+// whether an ancestor was marked as `sensitive`, not just the field, because of cases like
+// `TlsContext::private_key`, which is of type `core.DataSource` rather than `string`.
+void redactInPlace(Protobuf::Message* message, bool ancestor_is_sensitive) {
+  // If the message is an `Any`, we have to first unpack it to its original type to redact it...
+  auto* any = dynamic_cast<ProtobufWkt::Any*>(message);
+  if (any != nullptr) {
+    Protobuf::DynamicMessageFactory message_factory;
+    auto typed_message = typeUrlToNewMessage(message_factory, any->type_url());
+    if (typed_message == nullptr) {
+      // If the type URL doesn't correspond to a known proto, give up redacting and treat this
+      // message the same as a `ProtobufWkt::Struct`. See the documented limitation on
+      // `MessageUtil::redact()` for more context.
+      ENVOY_LOG_MISC(warn, "Could not redact ProtobufWkt::Any with unknown type URL {}",
+                     any->type_url());
+      return;
+    }
+
+    any->UnpackTo(typed_message.get());
+    redactInPlace(typed_message.get(), ancestor_is_sensitive);
+    any->PackFrom(*typed_message);
+    return;
+  }
+
+  // If the message is a `TypedStruct`, it also contains a `type_url` and can be handled the same
+  // way as `Any` above.
+  auto* typed_struct = dynamic_cast<udpa::type::v1::TypedStruct*>(message);
+  if (typed_struct != nullptr) {
+    Protobuf::DynamicMessageFactory message_factory;
+    auto typed_message = typeUrlToNewMessage(message_factory, typed_struct->type_url());
+    if (typed_message == nullptr) {
+      // If the type URL doesn't correspond to a known proto, give up redacting and treat this
+      // message the same as a `ProtobufWkt::Struct`. See the documented limitation on
+      // `MessageUtil::redact()` for more context.
+      ENVOY_LOG_MISC(warn, "Could not redact udpa::type::v1::TypedStruct with unknown type URL {}",
+                     typed_struct->type_url());
+      return;
+    }
+
+    MessageUtil::jsonConvert(typed_struct->value(), ProtobufMessage::getNullValidationVisitor(),
+                             *typed_message);
+    redactInPlace(typed_message.get(), ancestor_is_sensitive);
+    MessageUtil::jsonConvert(*typed_message, *(typed_struct->mutable_value()));
+    return;
+  }
+
+  // Otherwise, use reflection to traverse all populated fields of this message...
+  const auto* reflection = message->GetReflection();
+  std::vector<const Protobuf::FieldDescriptor*> field_descriptors;
+  reflection->ListFields(*message, &field_descriptors);
+  for (const auto* field_descriptor : field_descriptors) {
+    // Redact if this field or any of its ancestors have the `sensitive` option set.
+    const bool sensitive = ancestor_is_sensitive ||
+                           field_descriptor->options().GetExtension(udpa::annotations::sensitive);
+
+    if (field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_MESSAGE) {
+      // Recursive case: traverse message fields.
+      if (field_descriptor->is_repeated()) {
+        const int field_size = reflection->FieldSize(*message, field_descriptor);
+        for (int i = 0; i < field_size; ++i) {
+          redactInPlace(reflection->MutableRepeatedMessage(message, field_descriptor, i),
+                        sensitive);
+        }
+      } else {
+        redactInPlace(reflection->MutableMessage(message, field_descriptor), sensitive);
+      }
+    } else if (sensitive) {
+      // Base case: replace strings with "[redacted]" and clear all others.
+      if (field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_STRING) {
+        if (field_descriptor->is_repeated()) {
+          const int field_size = reflection->FieldSize(*message, field_descriptor);
+          for (int i = 0; i < field_size; ++i) {
+            reflection->SetRepeatedString(message, field_descriptor, i, "[redacted]");
+          }
+        } else {
+          reflection->SetString(message, field_descriptor, "[redacted]");
+        }
+      } else {
+        reflection->ClearField(message, field_descriptor);
+      }
+    }
+  }
+}
+
+} // namespace
+
+std::unique_ptr<Protobuf::Message> MessageUtil::redact(const Protobuf::Message& message) {
+  // 1. Clone the original message. New() will return an empty message of the correct dynamic type.
+  std::unique_ptr<Protobuf::Message> redacted(message.New());
+  redacted->MergeFrom(message);
+
+  // 2. Redact the cloned message.
+  redactInPlace(redacted.get(), /* ancestor_is_sensitive = */ false);
+  return redacted;
+}
+
 ProtobufWkt::Value ValueUtil::loadFromYaml(const std::string& yaml) {
   try {
     return parseYamlNode(YAML::Load(yaml));

source/common/protobuf/utility.h

@@ -361,6 +361,28 @@ class MessageUtil {
    * @param code the protobuf error code
    */
   static std::string CodeEnumToString(ProtobufUtil::error::Code code);
+
+  /**
+   * Makes a copy of a source message with all sensitive data (that is, fields annotated as
+   * `udpa.annotations.sensitive`) redacted for display. String-typed fields annotated as
+   * `sensitive` will be replaced with the string "[redacted]", primitive-typed fields (including
+   * enums) will be cleared, and message-typed fields will be traversed recursively to redact
+   * their contents.
+   *
+   * LIMITATION: This works properly for strongly-typed messages, as well as for messages packed in
+   * a `ProtobufWkt::Any` with a `type_url` corresponding to a proto that was compiled into the
+   * Envoy binary. However it does not work for messages encoded as `ProtobufWkt::Struct`, since
+   * structs are missing the "sensitive" annotations that this function expects. Similarly, it fails
+   * for messages encoded as `ProtobufWkt::Any` with a `type_url` that isn't registered with the
+   * binary. If you're working with struct-typed messages, including those that might be hiding
+   * within strongly-typed messages, please reify them to strongly-typed messages using
+   * `MessageUtil::jsonConvert()` before calling `MessageUtil::redact()`.
+   *
+   * @param message original message to copy from.
+   * @return cloned message of the same dynamic type as the original, with all sensitive data
+   * redacted.
+   */
+  static std::unique_ptr<Protobuf::Message> redact(const Protobuf::Message& message);
 };
 
 class ValueUtil {

source/server/http/admin.cc

@@ -541,9 +541,10 @@ Http::Code AdminImpl::handlerConfigDump(absl::string_view, Http::HeaderMap& resp
     auto& any_message = *(dump.add_configs());
     any_message.PackFrom(*message);
   }
+  const auto redacted = MessageUtil::redact(dump);
 
   response_headers.setReferenceContentType(Http::Headers::get().ContentTypeValues.Json);
-  response.add(MessageUtil::getJsonStringFromMessage(dump, true)); // pretty-print
+  response.add(MessageUtil::getJsonStringFromMessage(*redacted, true)); // pretty-print
   return Http::Code::OK;
 }
 

test/common/protobuf/BUILD

@@ -30,6 +30,7 @@ envoy_cc_test(
         "//test/mocks/protobuf:protobuf_mocks",
         "//test/mocks/server:server_mocks",
         "//test/proto:deprecated_proto_cc_proto",
+        "//test/proto:sensitive_proto_cc_proto",
         "//test/test_common:environment_lib",
         "//test/test_common:logging_lib",
         "//test/test_common:utility_lib",