diff --git a/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto b/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto index 2c8c606d3f863..5e70bbfce46f1 100644 --- a/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto +++ b/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto @@ -20,6 +20,21 @@ message Config { enum ValueType { STRING = 0; NUMBER = 1; + + // The value is a serialized `protobuf.Value + // `_. + PROTOBUF_VALUE = 2; + } + + // ValueEncode defines the encoding algorithm. + enum ValueEncode { + // The value is not encoded. + NONE = 0; + + // The value is encoded in `Base64 `_. + // Note: this is mostly used for STRING and PROTOBUF_VALUE to escape the + // non-ASCII characters in the header. + BASE64 = 1; } message KeyValuePair { @@ -40,6 +55,10 @@ message Config { // The value's type — defaults to string. ValueType type = 4; + + // How is the value encoded, default is NONE (not encoded). + // The value will be decoded accordingly before storing to metadata. + ValueEncode encode = 5; } // A Rule defines what metadata to apply when a header is present or missing. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 758247d95cf11..1bd70153bf63c 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -9,6 +9,7 @@ Version history * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * fault: added overrides for default runtime keys in :ref:`HTTPFault ` filter. * grpc-json: added support for :ref:`ignoring unknown query parameters`. +* header to metadata: added :ref:`PROTOBUF_VALUE ` and :ref:`ValueEncode ` to support protobuf Value and Base64 encoding. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`HTTP inspector listener filter `. diff --git a/source/extensions/filters/http/header_to_metadata/BUILD b/source/extensions/filters/http/header_to_metadata/BUILD index b31109c6333c4..f0ffbec64f5cd 100644 --- a/source/extensions/filters/http/header_to_metadata/BUILD +++ b/source/extensions/filters/http/header_to_metadata/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( hdrs = ["header_to_metadata_filter.h"], deps = [ "//include/envoy/server:filter_config_interface", + "//source/common/common:base64_lib", "//source/extensions/filters/http:well_known_names", "@envoy_api//envoy/config/filter/http/header_to_metadata/v2:header_to_metadata_cc", ], diff --git a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc index 8d0ccee24e817..6c4379e7a9d80 100644 --- a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc +++ b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc @@ -1,5 +1,6 @@ #include "extensions/filters/http/header_to_metadata/header_to_metadata_filter.h" +#include "common/common/base64.h" #include "common/config/well_known_names.h" #include "common/protobuf/protobuf.h" @@ -12,11 +13,6 @@ namespace Envoy { namespace Extensions { namespace HttpFilters { namespace HeaderToMetadataFilter { -namespace { - -const uint32_t MAX_HEADER_VALUE_LEN = 100; - -} // namespace Config::Config(const envoy::config::filter::http::header_to_metadata::v2::Config config) { request_set_ = Config::configToVector(config.request_rules(), request_rules_); @@ -83,7 +79,7 @@ void HeaderToMetadataFilter::setEncoderFilterCallbacks( bool HeaderToMetadataFilter::addMetadata(StructMap& map, const std::string& meta_namespace, const std::string& key, absl::string_view value, - ValueType type) const { + ValueType type, ValueEncode encode) const { ProtobufWkt::Value val; if (value.empty()) { @@ -98,14 +94,23 @@ bool HeaderToMetadataFilter::addMetadata(StructMap& map, const std::string& meta return false; } + std::string decodedValue = std::string(value); + if (encode == envoy::config::filter::http::header_to_metadata::v2::Config_ValueEncode_BASE64) { + decodedValue = Base64::decodeWithoutPadding(value); + if (decodedValue.empty()) { + ENVOY_LOG(debug, "Base64 decode failed"); + return false; + } + } + // Sane enough, add the key/value. switch (type) { case envoy::config::filter::http::header_to_metadata::v2::Config_ValueType_STRING: - val.set_string_value(std::string(value)); + val.set_string_value(std::move(decodedValue)); break; case envoy::config::filter::http::header_to_metadata::v2::Config_ValueType_NUMBER: { double dval; - if (absl::SimpleAtod(StringUtil::trim(value), &dval)) { + if (absl::SimpleAtod(StringUtil::trim(decodedValue), &dval)) { val.set_number_value(dval); } else { ENVOY_LOG(debug, "value to number conversion failed"); @@ -113,6 +118,13 @@ bool HeaderToMetadataFilter::addMetadata(StructMap& map, const std::string& meta } break; } + case envoy::config::filter::http::header_to_metadata::v2::Config_ValueType_PROTOBUF_VALUE: { + if (!val.ParseFromString(decodedValue)) { + ENVOY_LOG(debug, "parse from decoded string failed"); + return false; + } + break; + } default: ENVOY_LOG(debug, "unknown value type"); return false; @@ -152,7 +164,8 @@ void HeaderToMetadataFilter::writeHeaderToMetadata(Http::HeaderMap& headers, if (!value.empty()) { const auto& nspace = decideNamespace(keyval.metadata_namespace()); - addMetadata(structs_by_namespace, nspace, keyval.key(), value, keyval.type()); + addMetadata(structs_by_namespace, nspace, keyval.key(), value, keyval.type(), + keyval.encode()); } else { ENVOY_LOG(debug, "value is empty, not adding metadata"); } @@ -166,7 +179,8 @@ void HeaderToMetadataFilter::writeHeaderToMetadata(Http::HeaderMap& headers, if (!keyval.value().empty()) { const auto& nspace = decideNamespace(keyval.metadata_namespace()); - addMetadata(structs_by_namespace, nspace, keyval.key(), keyval.value(), keyval.type()); + addMetadata(structs_by_namespace, nspace, keyval.key(), keyval.value(), keyval.type(), + keyval.encode()); } else { ENVOY_LOG(debug, "value is empty, not adding metadata"); } diff --git a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h index e10da6e3eba95..297af97d2bc55 100644 --- a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h +++ b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h @@ -18,8 +18,12 @@ namespace HeaderToMetadataFilter { using Rule = envoy::config::filter::http::header_to_metadata::v2::Config::Rule; using ValueType = envoy::config::filter::http::header_to_metadata::v2::Config::ValueType; +using ValueEncode = envoy::config::filter::http::header_to_metadata::v2::Config::ValueEncode; using HeaderToMetadataRules = std::vector>; +// TODO(yangminzhu): Make MAX_HEADER_VALUE_LEN configurable. +const uint32_t MAX_HEADER_VALUE_LEN = 8 * 1024; + /** * Encapsulates the filter configuration with STL containers and provides an area for any custom * configuration logic. @@ -116,8 +120,8 @@ class HeaderToMetadataFilter : public Http::StreamFilter, */ void writeHeaderToMetadata(Http::HeaderMap& headers, const HeaderToMetadataRules& rules, Http::StreamFilterCallbacks& callbacks); - bool addMetadata(StructMap&, const std::string&, const std::string&, absl::string_view, - ValueType) const; + bool addMetadata(StructMap&, const std::string&, const std::string&, absl::string_view, ValueType, + ValueEncode) const; const std::string& decideNamespace(const std::string& nspace) const; }; diff --git a/test/extensions/filters/http/header_to_metadata/BUILD b/test/extensions/filters/http/header_to_metadata/BUILD index 0c2b80bff21bb..d447d310c9fe2 100644 --- a/test/extensions/filters/http/header_to_metadata/BUILD +++ b/test/extensions/filters/http/header_to_metadata/BUILD @@ -16,6 +16,7 @@ envoy_extension_cc_test( srcs = ["header_to_metadata_filter_test.cc"], extension_name = "envoy.filters.http.header_to_metadata", deps = [ + "//source/common/common:base64_lib", "//source/extensions/filters/http/header_to_metadata:header_to_metadata_filter_lib", "//test/mocks/server:server_mocks", ], diff --git a/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc b/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc index 1cfa6acfe73f4..9674e22e8217a 100644 --- a/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc +++ b/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc @@ -1,4 +1,6 @@ +#include "common/common/base64.h" #include "common/http/header_map_impl.h" +#include "common/protobuf/protobuf.h" #include "extensions/filters/http/header_to_metadata/header_to_metadata_filter.h" @@ -68,6 +70,15 @@ MATCHER_P(MapEqNum, rhs, "") { return true; } +MATCHER_P(MapEqValue, rhs, "") { + const ProtobufWkt::Struct& obj = arg; + EXPECT_TRUE(!rhs.empty()); + for (auto const& entry : rhs) { + EXPECT_TRUE(TestUtility::protoEqual(obj.fields().at(entry.first), entry.second)); + } + return true; +} + /** * Basic use-case. */ @@ -154,6 +165,110 @@ TEST_F(HeaderToMetadataTest, NumberTypeTest) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); } +/** + * Test the Base64 encoded value gets written as a string. + */ +TEST_F(HeaderToMetadataTest, StringTypeInBase64UrlTest) { + const std::string response_config_yaml = R"EOF( +response_rules: + - header: x-authenticated + on_header_present: + key: auth + type: STRING + encode: BASE64 +)EOF"; + initializeFilter(response_config_yaml); + std::string data = "Non-ascii-characters"; + const auto encoded = Base64::encode(data.c_str(), data.size()); + Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", encoded}}; + std::map expected = {{"auth", data}}; + Http::TestHeaderMapImpl empty_headers; + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + EXPECT_CALL(req_info_, + setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEq(expected))); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); +} + +/** + * Test the Base64 encoded protobuf value gets written as a protobuf value. + */ +TEST_F(HeaderToMetadataTest, ProtobufValueTypeInBase64UrlTest) { + const std::string response_config_yaml = R"EOF( +response_rules: + - header: x-authenticated + on_header_present: + key: auth + type: PROTOBUF_VALUE + encode: BASE64 +)EOF"; + initializeFilter(response_config_yaml); + + ProtobufWkt::Value value; + auto* s = value.mutable_struct_value(); + + ProtobufWkt::Value v; + v.set_string_value("blafoo"); + (*s->mutable_fields())["k1"] = v; + v.set_number_value(2019.07); + (*s->mutable_fields())["k2"] = v; + v.set_bool_value(true); + (*s->mutable_fields())["k3"] = v; + + std::string data; + ASSERT_TRUE(value.SerializeToString(&data)); + const auto encoded = Base64::encode(data.c_str(), data.size()); + Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", encoded}}; + std::map expected = {{"auth", value}}; + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + EXPECT_CALL(req_info_, + setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEqValue(expected))); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); +} + +/** + * Test bad Base64 encoding is not written. + */ +TEST_F(HeaderToMetadataTest, ProtobufValueTypeInBadBase64UrlTest) { + const std::string response_config_yaml = R"EOF( +response_rules: + - header: x-authenticated + on_header_present: + key: auth + type: PROTOBUF_VALUE + encode: BASE64 +)EOF"; + initializeFilter(response_config_yaml); + Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", "invalid"}}; + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); +} + +/** + * Test the bad protobuf value is not written. + */ +TEST_F(HeaderToMetadataTest, BadProtobufValueTypeInBase64UrlTest) { + const std::string response_config_yaml = R"EOF( +response_rules: + - header: x-authenticated + on_header_present: + key: auth + type: PROTOBUF_VALUE + encode: BASE64 +)EOF"; + initializeFilter(response_config_yaml); + std::string data = "invalid"; + const auto encoded = Base64::encode(data.c_str(), data.size()); + Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", encoded}}; + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); +} + /** * Headers not present. */ @@ -221,7 +336,8 @@ TEST_F(HeaderToMetadataTest, EmptyHeaderValue) { */ TEST_F(HeaderToMetadataTest, HeaderValueTooLong) { initializeFilter(request_config_yaml); - Http::TestHeaderMapImpl incoming_headers{{"X-VERSION", std::string(101, 'x')}}; + auto length = Envoy::Extensions::HttpFilters::HeaderToMetadataFilter::MAX_HEADER_VALUE_LEN + 1; + Http::TestHeaderMapImpl incoming_headers{{"X-VERSION", std::string(length, 'x')}}; EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0);