header to metadata: add base64 encode and protobuf value type

api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto

@@ -20,6 +20,21 @@ message Config {
   enum ValueType {
     STRING = 0;
     NUMBER = 1;
+
+    // The value is a serialized `protobuf.Value
+    // <https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/struct.proto#L62>`_.
+    PROTOBUF_VALUE = 2;
+  }
+
+  // ValueEncode defines the encoding algorithm.
+  enum ValueEncode {
+    // The value is not encoded.
+    NONE = 0;
+
+    // The value is encoded in `Base64 <https://tools.ietf.org/html/rfc4648#section-4>`_.
+    // Note: this is mostly used for STRING and PROTOBUF_VALUE to escape the
+    // non-ASCII characters in the header.
+    BASE64 = 1;
   }
 
   message KeyValuePair {
@@ -40,6 +55,10 @@ message Config {
 
     // The value's type — defaults to string.
     ValueType type = 4;
+
+    // How is the value encoded, default is NONE (not encoded).
+    // The value will be decoded accordingly before storing to metadata.
+    ValueEncode encode = 5;
   }
 
   // A Rule defines what metadata to apply when a header is present or missing.

docs/root/intro/version_history.rst

@@ -9,6 +9,7 @@ Version history
 * config: changed the default value of :ref:`initial_fetch_timeout <envoy_api_field_core.ConfigSource.initial_fetch_timeout>` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process <arch_overview_initialization>` for more details.
 * fault: added overrides for default runtime keys in :ref:`HTTPFault <envoy_api_msg_config.filter.http.fault.v2.HTTPFault>` filter.
 * grpc-json: added support for :ref:`ignoring unknown query parameters<envoy_api_field_config.filter.http.transcoder.v2.GrpcJsonTranscoder.ignore_unknown_query_parameters>`.
+* header to metadata: added :ref:`PROTOBUF_VALUE <envoy_api_enum_value_config.filter.http.header_to_metadata.v2.Config.ValueType.PROTOBUF_VALUE>` and :ref:`ValueEncode <envoy_api_enum_config.filter.http.header_to_metadata.v2.Config.ValueEncode>` to support protobuf Value and Base64 encoding.
 * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`.
 * http: added the ability to :ref:`merge adjacent slashes<envoy_api_field_config.filter.network.http_connection_manager.v2.HttpConnectionManager.merge_slashes>` in the path.
 * listeners: added :ref:`HTTP inspector listener filter <config_listener_filters_http_inspector>`.

source/extensions/filters/http/header_to_metadata/BUILD

@@ -17,6 +17,7 @@ envoy_cc_library(
     hdrs = ["header_to_metadata_filter.h"],
     deps = [
         "//include/envoy/server:filter_config_interface",
+        "//source/common/common:base64_lib",
         "//source/extensions/filters/http:well_known_names",
         "@envoy_api//envoy/config/filter/http/header_to_metadata/v2:header_to_metadata_cc",
     ],

source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc

@@ -1,5 +1,6 @@
 #include "extensions/filters/http/header_to_metadata/header_to_metadata_filter.h"
 
+#include "common/common/base64.h"
 #include "common/config/well_known_names.h"
 #include "common/protobuf/protobuf.h"
 
@@ -12,11 +13,6 @@ namespace Envoy {
 namespace Extensions {
 namespace HttpFilters {
 namespace HeaderToMetadataFilter {
-namespace {
-
-const uint32_t MAX_HEADER_VALUE_LEN = 100;
-
-} // namespace
 
 Config::Config(const envoy::config::filter::http::header_to_metadata::v2::Config config) {
   request_set_ = Config::configToVector(config.request_rules(), request_rules_);
@@ -83,7 +79,7 @@ void HeaderToMetadataFilter::setEncoderFilterCallbacks(
 
 bool HeaderToMetadataFilter::addMetadata(StructMap& map, const std::string& meta_namespace,
                                          const std::string& key, absl::string_view value,
-                                         ValueType type) const {
+                                         ValueType type, ValueEncode encode) const {
   ProtobufWkt::Value val;
 
   if (value.empty()) {
@@ -98,21 +94,37 @@ bool HeaderToMetadataFilter::addMetadata(StructMap& map, const std::string& meta
     return false;
   }
 
+  std::string decodedValue = std::string(value);
+  if (encode == envoy::config::filter::http::header_to_metadata::v2::Config_ValueEncode_BASE64) {
+    decodedValue = Base64::decodeWithoutPadding(value);
+    if (decodedValue.empty()) {
+      ENVOY_LOG(debug, "Base64 decode failed");
+      return false;
+    }
+  }
+
   // Sane enough, add the key/value.
   switch (type) {
   case envoy::config::filter::http::header_to_metadata::v2::Config_ValueType_STRING:
-    val.set_string_value(std::string(value));
+    val.set_string_value(std::move(decodedValue));
     break;
   case envoy::config::filter::http::header_to_metadata::v2::Config_ValueType_NUMBER: {
     double dval;
-    if (absl::SimpleAtod(StringUtil::trim(value), &dval)) {
+    if (absl::SimpleAtod(StringUtil::trim(decodedValue), &dval)) {
       val.set_number_value(dval);
     } else {
       ENVOY_LOG(debug, "value to number conversion failed");
       return false;
     }
     break;
   }
+  case envoy::config::filter::http::header_to_metadata::v2::Config_ValueType_PROTOBUF_VALUE: {
+    if (!val.ParseFromString(decodedValue)) {
+      ENVOY_LOG(debug, "parse from decoded string failed");
+      return false;
+    }
+    break;
+  }
   default:
     ENVOY_LOG(debug, "unknown value type");
     return false;
@@ -152,7 +164,8 @@ void HeaderToMetadataFilter::writeHeaderToMetadata(Http::HeaderMap& headers,
 
       if (!value.empty()) {
         const auto& nspace = decideNamespace(keyval.metadata_namespace());
-        addMetadata(structs_by_namespace, nspace, keyval.key(), value, keyval.type());
+        addMetadata(structs_by_namespace, nspace, keyval.key(), value, keyval.type(),
+                    keyval.encode());
       } else {
         ENVOY_LOG(debug, "value is empty, not adding metadata");
       }
@@ -166,7 +179,8 @@ void HeaderToMetadataFilter::writeHeaderToMetadata(Http::HeaderMap& headers,
 
       if (!keyval.value().empty()) {
         const auto& nspace = decideNamespace(keyval.metadata_namespace());
-        addMetadata(structs_by_namespace, nspace, keyval.key(), keyval.value(), keyval.type());
+        addMetadata(structs_by_namespace, nspace, keyval.key(), keyval.value(), keyval.type(),
+                    keyval.encode());
       } else {
         ENVOY_LOG(debug, "value is empty, not adding metadata");
       }

source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h

@@ -18,8 +18,12 @@ namespace HeaderToMetadataFilter {
 
 using Rule = envoy::config::filter::http::header_to_metadata::v2::Config::Rule;
 using ValueType = envoy::config::filter::http::header_to_metadata::v2::Config::ValueType;
+using ValueEncode = envoy::config::filter::http::header_to_metadata::v2::Config::ValueEncode;
 using HeaderToMetadataRules = std::vector<std::pair<Http::LowerCaseString, Rule>>;
 
+// TODO(yangminzhu): Make MAX_HEADER_VALUE_LEN configurable.
+const uint32_t MAX_HEADER_VALUE_LEN = 8 * 1024;
+
 /**
  *  Encapsulates the filter configuration with STL containers and provides an area for any custom
  *  configuration logic.
@@ -116,8 +120,8 @@ class HeaderToMetadataFilter : public Http::StreamFilter,
    */
   void writeHeaderToMetadata(Http::HeaderMap& headers, const HeaderToMetadataRules& rules,
                              Http::StreamFilterCallbacks& callbacks);
-  bool addMetadata(StructMap&, const std::string&, const std::string&, absl::string_view,
-                   ValueType) const;
+  bool addMetadata(StructMap&, const std::string&, const std::string&, absl::string_view, ValueType,
+                   ValueEncode) const;
   const std::string& decideNamespace(const std::string& nspace) const;
 };
 

test/extensions/filters/http/header_to_metadata/BUILD

@@ -16,6 +16,7 @@ envoy_extension_cc_test(
     srcs = ["header_to_metadata_filter_test.cc"],
     extension_name = "envoy.filters.http.header_to_metadata",
     deps = [
+        "//source/common/common:base64_lib",
         "//source/extensions/filters/http/header_to_metadata:header_to_metadata_filter_lib",
         "//test/mocks/server:server_mocks",
     ],

test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc

@@ -1,4 +1,6 @@
+#include "common/common/base64.h"
 #include "common/http/header_map_impl.h"
+#include "common/protobuf/protobuf.h"
 
 #include "extensions/filters/http/header_to_metadata/header_to_metadata_filter.h"
 
@@ -68,6 +70,15 @@ MATCHER_P(MapEqNum, rhs, "") {
   return true;
 }
 
+MATCHER_P(MapEqValue, rhs, "") {
+  const ProtobufWkt::Struct& obj = arg;
+  EXPECT_TRUE(!rhs.empty());
+  for (auto const& entry : rhs) {
+    EXPECT_TRUE(TestUtility::protoEqual(obj.fields().at(entry.first), entry.second));
+  }
+  return true;
+}
+
 /**
  * Basic use-case.
  */
@@ -154,6 +165,110 @@ TEST_F(HeaderToMetadataTest, NumberTypeTest) {
   EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false));
 }
 
+/**
+ * Test the Base64 encoded value gets written as a string.
+ */
+TEST_F(HeaderToMetadataTest, StringTypeInBase64UrlTest) {
+  const std::string response_config_yaml = R"EOF(
+response_rules:
+  - header: x-authenticated
+    on_header_present:
+      key: auth
+      type: STRING
+      encode: BASE64
+)EOF";
+  initializeFilter(response_config_yaml);
+  std::string data = "Non-ascii-characters";
+  const auto encoded = Base64::encode(data.c_str(), data.size());
+  Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", encoded}};
+  std::map<std::string, std::string> expected = {{"auth", data}};
+  Http::TestHeaderMapImpl empty_headers;
+
+  EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_));
+  EXPECT_CALL(req_info_,
+              setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEq(expected)));
+  EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false));
+}
+
+/**
+ * Test the Base64 encoded protobuf value gets written as a protobuf value.
+ */
+TEST_F(HeaderToMetadataTest, ProtobufValueTypeInBase64UrlTest) {
+  const std::string response_config_yaml = R"EOF(
+response_rules:
+  - header: x-authenticated
+    on_header_present:
+      key: auth
+      type: PROTOBUF_VALUE
+      encode: BASE64
+)EOF";
+  initializeFilter(response_config_yaml);
+
+  ProtobufWkt::Value value;
+  auto* s = value.mutable_struct_value();
+
+  ProtobufWkt::Value v;
+  v.set_string_value("blafoo");
+  (*s->mutable_fields())["k1"] = v;
+  v.set_number_value(2019.07);
+  (*s->mutable_fields())["k2"] = v;
+  v.set_bool_value(true);
+  (*s->mutable_fields())["k3"] = v;
+
+  std::string data;
+  ASSERT_TRUE(value.SerializeToString(&data));
+  const auto encoded = Base64::encode(data.c_str(), data.size());
+  Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", encoded}};
+  std::map<std::string, ProtobufWkt::Value> expected = {{"auth", value}};
+
+  EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_));
+  EXPECT_CALL(req_info_,
+              setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEqValue(expected)));
+  EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false));
+}
+
+/**
+ * Test bad Base64 encoding is not written.
+ */
+TEST_F(HeaderToMetadataTest, ProtobufValueTypeInBadBase64UrlTest) {
+  const std::string response_config_yaml = R"EOF(
+response_rules:
+  - header: x-authenticated
+    on_header_present:
+      key: auth
+      type: PROTOBUF_VALUE
+      encode: BASE64
+)EOF";
+  initializeFilter(response_config_yaml);
+  Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", "invalid"}};
+
+  EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_));
+  EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0);
+  EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false));
+}
+
+/**
+ * Test the bad protobuf value is not written.
+ */
+TEST_F(HeaderToMetadataTest, BadProtobufValueTypeInBase64UrlTest) {
+  const std::string response_config_yaml = R"EOF(
+response_rules:
+  - header: x-authenticated
+    on_header_present:
+      key: auth
+      type: PROTOBUF_VALUE
+      encode: BASE64
+)EOF";
+  initializeFilter(response_config_yaml);
+  std::string data = "invalid";
+  const auto encoded = Base64::encode(data.c_str(), data.size());
+  Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", encoded}};
+
+  EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_));
+  EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0);
+  EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false));
+}
+
 /**
  * Headers not present.
  */
@@ -221,7 +336,8 @@ TEST_F(HeaderToMetadataTest, EmptyHeaderValue) {
  */
 TEST_F(HeaderToMetadataTest, HeaderValueTooLong) {
   initializeFilter(request_config_yaml);
-  Http::TestHeaderMapImpl incoming_headers{{"X-VERSION", std::string(101, 'x')}};
+  auto length = Envoy::Extensions::HttpFilters::HeaderToMetadataFilter::MAX_HEADER_VALUE_LEN + 1;
+  Http::TestHeaderMapImpl incoming_headers{{"X-VERSION", std::string(length, 'x')}};
 
   EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_));
   EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0);