diff --git a/RAW_RELEASE_NOTES.md b/RAW_RELEASE_NOTES.md index ffc374ffdc359..478052736c5c6 100644 --- a/RAW_RELEASE_NOTES.md +++ b/RAW_RELEASE_NOTES.md @@ -44,3 +44,4 @@ final version. * Added support for building envoy with exported symbols This change allows scripts loaded with the lua filter to load shared object libraries such as those installed via luarocks. * Added support for more granular weighted cluster routing by allowing the total weight to be specified in configuration. +* Added support for custom request/response headers with mixed static and dynamic values. diff --git a/source/common/common/macros.h b/source/common/common/macros.h index b947d712f4344..8a16a19a3c1fe 100644 --- a/source/common/common/macros.h +++ b/source/common/common/macros.h @@ -7,6 +7,11 @@ namespace Envoy { */ #define ARRAY_SIZE(X) (sizeof(X) / sizeof(X[0])) +/** + * @return the length of a static string literal, e.g. STATIC_STRLEN("foo") == 3. + */ +#define STATIC_STRLEN(X) (sizeof(X) - 1) + /** * Helper macros from enum to string macros. */ diff --git a/source/common/router/header_formatter.cc b/source/common/router/header_formatter.cc index c570e8d2825a5..05d4e3272089e 100644 --- a/source/common/router/header_formatter.cc +++ b/source/common/router/header_formatter.cc @@ -18,17 +18,17 @@ namespace Router { namespace { -std::string formatUpstreamMetadataParseException(const std::string& params, +std::string formatUpstreamMetadataParseException(absl::string_view params, const EnvoyException* cause = nullptr) { std::string reason; if (cause != nullptr) { reason = fmt::format(", because {}", cause->what()); } - return fmt::format("Incorrect header configuration. Expected format " + return fmt::format("Invalid header configuration. Expected format " "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " "UPSTREAM_METADATA{}{}", - params, reason); + std::string(params), reason); } // Parses the parameters for UPSTREAM_METADATA and returns a function suitable for accessing the @@ -36,15 +36,17 @@ std::string formatUpstreamMetadataParseException(const std::string& params, // (["a", "b", "c"]) // There must be at least 2 array elements (a metadata namespace and at least 1 key). std::function -parseUpstreamMetadataField(const std::string& params_str) { +parseUpstreamMetadataField(absl::string_view params_str) { + params_str = StringUtil::trim(params_str); if (params_str.empty() || params_str.front() != '(' || params_str.back() != ')') { throw EnvoyException(formatUpstreamMetadataParseException(params_str)); } + absl::string_view json = params_str.substr(1, params_str.size() - 2); // trim parens + std::vector params; try { - Json::ObjectSharedPtr parsed_params = - Json::Factory::loadFromString(StringUtil::subspan(params_str, 1, params_str.size() - 1)); + Json::ObjectSharedPtr parsed_params = Json::Factory::loadFromString(std::string(json)); for (const auto& param : parsed_params->asObjectArray()) { params.emplace_back(param->asString()); @@ -113,7 +115,7 @@ parseUpstreamMetadataField(const std::string& params_str) { } // namespace -RequestInfoHeaderFormatter::RequestInfoHeaderFormatter(const std::string& field_name, bool append) +RequestInfoHeaderFormatter::RequestInfoHeaderFormatter(absl::string_view field_name, bool append) : append_(append) { if (field_name == "PROTOCOL") { field_extractor_ = [](const Envoy::RequestInfo::RequestInfo& request_info) { @@ -125,11 +127,12 @@ RequestInfoHeaderFormatter::RequestInfoHeaderFormatter(const std::string& field_ return RequestInfo::Utility::formatDownstreamAddressNoPort( *request_info.downstreamRemoteAddress()); }; - } else if (StringUtil::startsWith(field_name.c_str(), "UPSTREAM_METADATA")) { + } else if (field_name.find_first_of("UPSTREAM_METADATA") == 0) { field_extractor_ = - parseUpstreamMetadataField(field_name.substr(sizeof("UPSTREAM_METADATA") - 1)); + parseUpstreamMetadataField(field_name.substr(STATIC_STRLEN("UPSTREAM_METADATA"))); } else { - throw EnvoyException(fmt::format("field '{}' not supported as custom header", field_name)); + throw EnvoyException( + fmt::format("field '{}' not supported as custom header", std::string(field_name))); } } diff --git a/source/common/router/header_formatter.h b/source/common/router/header_formatter.h index ebb2c54da64ed..cc99ddacb27e4 100644 --- a/source/common/router/header_formatter.h +++ b/source/common/router/header_formatter.h @@ -6,6 +6,8 @@ #include "envoy/access_log/access_log.h" +#include "absl/strings/string_view.h" + namespace Envoy { namespace Router { @@ -32,7 +34,7 @@ typedef std::unique_ptr HeaderFormatterPtr; */ class RequestInfoHeaderFormatter : public HeaderFormatter { public: - RequestInfoHeaderFormatter(const std::string& field_name, bool append); + RequestInfoHeaderFormatter(absl::string_view field_name, bool append); // HeaderFormatter::format const std::string format(const Envoy::RequestInfo::RequestInfo& request_info) const override; @@ -49,7 +51,7 @@ class RequestInfoHeaderFormatter : public HeaderFormatter { class PlainHeaderFormatter : public HeaderFormatter { public: PlainHeaderFormatter(const std::string& static_header_value, bool append) - : static_value_(static_header_value), append_(append){}; + : static_value_(static_header_value), append_(append) {} // HeaderFormatter::format const std::string format(const Envoy::RequestInfo::RequestInfo&) const override { @@ -62,5 +64,28 @@ class PlainHeaderFormatter : public HeaderFormatter { const bool append_; }; +/** + * A formatter that produces a value by concatenating the results of multiple HeaderFormatters. + */ +class CompoundHeaderFormatter : public HeaderFormatter { +public: + CompoundHeaderFormatter(std::vector&& formatters, bool append) + : formatters_(std::move(formatters)), append_(append) {} + + // HeaderFormatter::format + const std::string format(const Envoy::RequestInfo::RequestInfo& request_info) const override { + std::string buf; + for (const auto& formatter : formatters_) { + buf += formatter->format(request_info); + } + return buf; + }; + bool append() const override { return append_; } + +private: + const std::vector formatters_; + const bool append_; +}; + } // namespace Router } // namespace Envoy diff --git a/source/common/router/header_parser.cc b/source/common/router/header_parser.cc index 2021d78d0962a..7f681ef672625 100644 --- a/source/common/router/header_parser.cc +++ b/source/common/router/header_parser.cc @@ -1,28 +1,207 @@ #include "common/router/header_parser.h" +#include + +#include +#include + +#include "common/common/assert.h" #include "common/protobuf/utility.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" + namespace Envoy { namespace Router { namespace { +enum class ParserState { + Literal, // processing literal data + VariableName, // consuming a %VAR% name + ExpectArray, // expect starting [ in %VAR([...])% + ExpectString, // expect starting " in array of strings + String, // consuming an array element string + ExpectArrayDelimiterOrEnd, // expect array delimiter (,) or end of array (]) + ExpectArgsEnd, // expect closing ) in %VAR(...)% + ExpectVariableEnd // expect closing % in %VAR(...)% +}; + +std::string unescape(absl::string_view sv) { return absl::StrReplaceAll(sv, {{"%%", "%"}}); } + +// Implements a state machine to parse custom headers. Each character of the custom header format +// is either literal text (with % escaped as %%) or part of a %VAR% or %VAR(["args"])% expression. +// The statement machine does minimal validation of the arguments (if any) and does not know the +// names of valid variables. Interpretation of the variable name and arguments is delegated to +// RequestInfoHeaderFormatter. HeaderFormatterPtr parseInternal(const envoy::api::v2::HeaderValueOption& header_value_option) { - const std::string& format = header_value_option.header().value(); const bool append = PROTOBUF_GET_WRAPPED_OR_DEFAULT(header_value_option, append, true); - if (format.find("%") == 0) { - const size_t last_occ_pos = format.rfind("%"); - if (last_occ_pos == std::string::npos || last_occ_pos <= 1) { - throw EnvoyException(fmt::format("Incorrect header configuration. Expected variable format " - "%%, actual format {}", - format)); + absl::string_view format(header_value_option.header().value()); + if (format.empty()) { + return std::make_unique("", append); + } + + std::vector formatters; + + size_t pos = 0, start = 0; + ParserState state = ParserState::Literal; + do { + const char ch = format[pos]; + const bool has_next_ch = (pos + 1) < format.size(); + + switch (state) { + case ParserState::Literal: + // Searching for start of %VARIABLE% expression. + if (ch != '%') { + break; + } + + if (!has_next_ch) { + throw EnvoyException( + fmt::format("Invalid header configuration. Un-escaped % at position {}", pos)); + } + + if (format[pos + 1] == '%') { + // Escaped %, skip next character. + pos++; + break; + } + + // Un-escaped %: start of variable name. Create a formatter for preceding characters, if + // any. + state = ParserState::VariableName; + if (pos > start) { + absl::string_view literal = format.substr(start, pos - start); + formatters.emplace_back(new PlainHeaderFormatter(unescape(literal), append)); + } + start = pos + 1; + break; + + case ParserState::VariableName: + // Consume "VAR" from "%VAR%" or "%VAR(...)%" + if (ch == '%') { + // Found complete variable name, add formatter. + formatters.emplace_back( + new RequestInfoHeaderFormatter(format.substr(start, pos - start), append)); + start = pos + 1; + state = ParserState::Literal; + break; + } + + if (ch == '(') { + // Variable with arguments, search for start of arg array. + state = ParserState::ExpectArray; + } + break; + + case ParserState::ExpectArray: + // Skip over whitespace searching for the start of JSON array args. + if (ch == '[') { + // Search for first argument string + state = ParserState::ExpectString; + } else if (!isspace(ch)) { + throw EnvoyException(fmt::format( + "Invalid header configuration. Expecting JSON array of arguments after '{}', but " + "found '{}'", + absl::StrCat(format.substr(start, pos - start)), ch)); + } + break; + + case ParserState::ExpectArrayDelimiterOrEnd: + // Skip over whitespace searching for a comma or close bracket. + if (ch == ',') { + state = ParserState::ExpectString; + } else if (ch == ']') { + state = ParserState::ExpectArgsEnd; + } else if (!isspace(ch)) { + throw EnvoyException(fmt::format( + "Invalid header configuration. Expecting ',', ']', or whitespace after '{}', but " + "found '{}'", + absl::StrCat(format.substr(start, pos - start)), ch)); + } + break; + + case ParserState::ExpectString: + // Skip over whitespace looking for the starting quote of a JSON string. + if (ch == '"') { + state = ParserState::String; + } else if (!isspace(ch)) { + throw EnvoyException(fmt::format( + "Invalid header configuration. Expecting '\"' or whitespace after '{}', but found '{}'", + absl::StrCat(format.substr(start, pos - start)), ch)); + } + break; + + case ParserState::String: + // Consume a JSON string (ignoring backslash-escaped chars). + if (ch == '\\') { + if (!has_next_ch) { + throw EnvoyException(fmt::format( + "Invalid header configuration. Un-terminated backslash in JSON string after '{}'", + absl::StrCat(format.substr(start, pos - start)))); + } + + // Skip escaped char. + pos++; + } else if (ch == '"') { + state = ParserState::ExpectArrayDelimiterOrEnd; + } + break; + + case ParserState::ExpectArgsEnd: + // Search for the closing paren of a %VAR(...)% expression. + if (ch == ')') { + state = ParserState::ExpectVariableEnd; + } else if (!isspace(ch)) { + throw EnvoyException(fmt::format( + "Invalid header configuration. Expecting ')' or whitespace after '{}', but found '{}'", + absl::StrCat(format.substr(start, pos - start)), ch)); + } + break; + + case ParserState::ExpectVariableEnd: + // Search for closing % of a %VAR(...)% expression + if (ch == '%') { + formatters.emplace_back( + new RequestInfoHeaderFormatter(format.substr(start, pos - start), append)); + start = pos + 1; + state = ParserState::Literal; + break; + } + + if (!isspace(ch)) { + throw EnvoyException(fmt::format( + "Invalid header configuration. Expecting '%' or whitespace after '{}', but found '{}'", + absl::StrCat(format.substr(start, pos - start)), ch)); + } + break; + + default: + NOT_REACHED; } - return HeaderFormatterPtr{ - new RequestInfoHeaderFormatter(format.substr(1, last_occ_pos - 1), append)}; - } else { - return HeaderFormatterPtr{new PlainHeaderFormatter(format, append)}; + } while (++pos < format.size()); + + if (state != ParserState::Literal) { + // Parsing terminated mid-variable. + throw EnvoyException( + fmt::format("Invalid header configuration. Un-terminated variable expression '{}'", + absl::StrCat(format.substr(start, pos - start)))); + } + + if (pos > start) { + // Trailing constant data. + absl::string_view literal = format.substr(start, pos - start); + formatters.emplace_back(new PlainHeaderFormatter(unescape(literal), append)); } + + ASSERT(formatters.size() > 0); + + if (formatters.size() == 1) { + return std::move(formatters[0]); + } + + return std::make_unique(std::move(formatters), append); } } // namespace diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 64e6de4013ae3..c90a981e7667d 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -3397,8 +3397,7 @@ TEST(CustomRequestHeadersTest, CustomHeaderWrongFormat) { NiceMock request_info; EXPECT_THROW_WITH_MESSAGE( ConfigImpl config(parseRouteConfigurationFromJson(json), runtime, cm, true), EnvoyException, - "Incorrect header configuration. Expected variable format %%, actual format " - "%CLIENT_IP"); + "Invalid header configuration. Un-terminated variable expression 'CLIENT_IP'"); } TEST(MetadataMatchCriteriaImpl, Create) { diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index 250abb9c96f52..bd1d51d69bbad 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -1,5 +1,6 @@ #include +#include "envoy/api/v2/base.pb.h" #include "envoy/http/protocol.h" #include "common/config/metadata.h" @@ -33,33 +34,45 @@ static envoy::api::v2::route::Route parseRouteFromV2Yaml(const std::string& yaml return route; } -TEST(RequestInfoHeaderFormatterTest, TestFormatWithClientIpVariable) { - NiceMock request_info; - const std::string variable = "CLIENT_IP"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("127.0.0.1", formatted_string); +class RequestInfoHeaderFormatterTest : public testing::Test { +public: + void testFormatting(const Envoy::RequestInfo::MockRequestInfo& request_info, + const std::string& variable, const std::string& expected_output) { + { + auto f = RequestInfoHeaderFormatter(variable, false); + const std::string formatted_string = f.format(request_info); + EXPECT_EQ(expected_output, formatted_string); + } + } + + void testFormatting(const std::string& variable, const std::string& expected_output) { + NiceMock request_info; + testFormatting(request_info, variable, expected_output); + } + + void testInvalidFormat(const std::string& variable) { + EXPECT_THROW_WITH_MESSAGE(RequestInfoHeaderFormatter(variable, false), EnvoyException, + fmt::format("field '{}' not supported as custom header", variable)); + } +}; + +TEST_F(RequestInfoHeaderFormatterTest, TestFormatWithClientIpVariable) { + testFormatting("CLIENT_IP", "127.0.0.1"); } -TEST(RequestInfoHeaderFormatterTest, TestFormatWithDownstreamRemoteAddressVariable) { - NiceMock request_info; - const std::string variable = "DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("127.0.0.1", formatted_string); +TEST_F(RequestInfoHeaderFormatterTest, TestFormatWithDownstreamRemoteAddressVariable) { + testFormatting("DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT", "127.0.0.1"); } -TEST(RequestInfoHeaderFormatterTest, TestFormatWithProtocolVariable) { +TEST_F(RequestInfoHeaderFormatterTest, TestFormatWithProtocolVariable) { NiceMock request_info; Optional protocol = Envoy::Http::Protocol::Http11; ON_CALL(request_info, protocol()).WillByDefault(ReturnRef(protocol)); - const std::string variable = "PROTOCOL"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("HTTP/1.1", formatted_string); + + testFormatting(request_info, "PROTOCOL", "HTTP/1.1"); } -TEST(RequestInfoFormatterTest, TestFormatWithUpstreamMetadataVariable) { +TEST_F(RequestInfoHeaderFormatterTest, TestFormatWithUpstreamMetadataVariable) { NiceMock request_info; std::shared_ptr> host( new NiceMock()); @@ -98,223 +111,314 @@ TEST(RequestInfoFormatterTest, TestFormatWithUpstreamMetadataVariable) { ON_CALL(*host, metadata()).WillByDefault(ReturnRef(metadata)); // Top-level value. - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"key\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("value", formatted_string); - } + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"key\"])", "value"); // Nested string value. - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"nested\", \"str_key\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("str_value", formatted_string); - } + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"str_key\"])", + "str_value"); // Boolean values. - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"nested\", \"bool_key1\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("true", formatted_string); - } - - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"nested\", \"bool_key2\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("false", formatted_string); - } + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"bool_key1\"])", + "true"); + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"bool_key2\"])", + "false"); // Number values. - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"nested\", \"num_key1\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("1", formatted_string); - } - - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"nested\", \"num_key2\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("3.14", formatted_string); - } + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"num_key1\"])", "1"); + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"num_key2\"])", + "3.14"); // Deeply nested value. - { - const std::string variable = - "UPSTREAM_METADATA([\"namespace\", \"nested\", \"struct_key\", \"deep_key\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("deep_value", formatted_string); - } + testFormatting(request_info, + "UPSTREAM_METADATA([\"namespace\", \"nested\", \"struct_key\", \"deep_key\"])", + "deep_value"); // Initial metadata lookup fails. - { - const std::string variable = "UPSTREAM_METADATA([\"wrong_namespace\", \"key\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("", formatted_string); - } - - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"not_found\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("", formatted_string); - } - - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"not_found\", \"key\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("", formatted_string); - } + testFormatting(request_info, "UPSTREAM_METADATA([\"wrong_namespace\", \"key\"])", ""); + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"not_found\"])", ""); + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"not_found\", \"key\"])", ""); // Nested metadata lookup fails. - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"nested\", \"not_found\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("", formatted_string); - } + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"not_found\"])", ""); // Nested metadata lookup returns non-struct intermediate value. - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"key\", \"invalid\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("", formatted_string); - } + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"key\", \"invalid\"])", ""); // Struct values are not rendered. - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"nested\", \"struct_key\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("", formatted_string); - } + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"struct_key\"])", + ""); // List values are not rendered. - { - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"nested\", \"list_key\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("", formatted_string); - } + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"nested\", \"list_key\"])", ""); } -TEST(RequestInfoFormatterTest, TestFormatWithUpstreamMetadataVariableMissingHost) { +TEST_F(RequestInfoHeaderFormatterTest, TestFormatWithUpstreamMetadataVariableMissingHost) { NiceMock request_info; std::shared_ptr> host; ON_CALL(request_info, upstreamHost()).WillByDefault(Return(host)); - const std::string variable = "UPSTREAM_METADATA([\"namespace\", \"key\"])"; - RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false); - const std::string formatted_string = requestInfoHeaderFormatter.format(request_info); - EXPECT_EQ("", formatted_string); -} - -TEST(RequestInfoHeaderFormatterTest, WrongVariableToFormat) { - NiceMock request_info; - const std::string variable = "INVALID_VARIABLE"; - EXPECT_THROW_WITH_MESSAGE(RequestInfoHeaderFormatter requestInfoHeaderFormatter(variable, false), - EnvoyException, - "field 'INVALID_VARIABLE' not supported as custom header"); + testFormatting(request_info, "UPSTREAM_METADATA([\"namespace\", \"key\"])", ""); } -TEST(RequestInfoHeaderFormatterTest, WrongFormatOnVariable) { - const std::string json = R"EOF( - { - "prefix": "/new_endpoint", - "prefix_rewrite": "/api/new_endpoint", - "cluster": "www2", - "request_headers_to_add": [ - { - "key": "x-client-ip", - "value": "%CLIENT_IP" - } - ] - } - )EOF"; - EXPECT_THROW_WITH_MESSAGE(Envoy::Router::HeaderParser::configure( - parseRouteFromJson(json).route().request_headers_to_add()), - EnvoyException, - "Incorrect header configuration. Expected variable format " - "%%, actual format %CLIENT_IP"); -} +TEST_F(RequestInfoHeaderFormatterTest, UnknownVariable) { testInvalidFormat("INVALID_VARIABLE"); } -TEST(RequestInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { +TEST_F(RequestInfoHeaderFormatterTest, WrongFormatOnUpstreamMetadataVariable) { // Invalid JSON. - EXPECT_THROW_WITH_MESSAGE( - RequestInfoHeaderFormatter requestInfoHeaderFormatter("UPSTREAM_METADATA(abcd)", false), - EnvoyException, - "Incorrect header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA(abcd), because JSON supplied is not valid. Error(offset 0, line 1): " - "Invalid value.\n"); + EXPECT_THROW_WITH_MESSAGE(RequestInfoHeaderFormatter("UPSTREAM_METADATA(abcd)", false), + EnvoyException, + "Invalid header configuration. Expected format " + "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " + "UPSTREAM_METADATA(abcd), because JSON supplied is not valid. " + "Error(offset 0, line 1): Invalid value.\n"); // No parameters. - EXPECT_THROW_WITH_MESSAGE( - RequestInfoHeaderFormatter requestInfoHeaderFormatter("UPSTREAM_METADATA", false), - EnvoyException, - "Incorrect header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA"); + EXPECT_THROW_WITH_MESSAGE(RequestInfoHeaderFormatter("UPSTREAM_METADATA", false), EnvoyException, + "Invalid header configuration. Expected format " + "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " + "UPSTREAM_METADATA"); - EXPECT_THROW_WITH_MESSAGE( - RequestInfoHeaderFormatter requestInfoHeaderFormatter("UPSTREAM_METADATA()", false), - EnvoyException, - "Incorrect header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA(), because JSON supplied is not valid. Error(offset 0, line 1): " - "The document is empty.\n"); + EXPECT_THROW_WITH_MESSAGE(RequestInfoHeaderFormatter("UPSTREAM_METADATA()", false), + EnvoyException, + "Invalid header configuration. Expected format " + "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " + "UPSTREAM_METADATA(), because JSON supplied is not valid. " + "Error(offset 0, line 1): The document is empty.\n"); // One parameter. - EXPECT_THROW_WITH_MESSAGE( - RequestInfoHeaderFormatter requestInfoHeaderFormatter("UPSTREAM_METADATA([\"ns\"])", false), - EnvoyException, - "Incorrect header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA([\"ns\"])"); + EXPECT_THROW_WITH_MESSAGE(RequestInfoHeaderFormatter("UPSTREAM_METADATA([\"ns\"])", false), + EnvoyException, + "Invalid header configuration. Expected format " + "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " + "UPSTREAM_METADATA([\"ns\"])"); // Missing close paren. - EXPECT_THROW_WITH_MESSAGE( - RequestInfoHeaderFormatter requestInfoHeaderFormatter("UPSTREAM_METADATA(", false), - EnvoyException, - "Incorrect header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA("); + EXPECT_THROW_WITH_MESSAGE(RequestInfoHeaderFormatter("UPSTREAM_METADATA(", false), EnvoyException, + "Invalid header configuration. Expected format " + "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " + "UPSTREAM_METADATA("); - EXPECT_THROW_WITH_MESSAGE( - RequestInfoHeaderFormatter requestInfoHeaderFormatter("UPSTREAM_METADATA([a,b,c,d]", false), - EnvoyException, - "Incorrect header configuration. Expected format " - "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " - "UPSTREAM_METADATA([a,b,c,d]"); + EXPECT_THROW_WITH_MESSAGE(RequestInfoHeaderFormatter("UPSTREAM_METADATA([a,b,c,d]", false), + EnvoyException, + "Invalid header configuration. Expected format " + "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " + "UPSTREAM_METADATA([a,b,c,d]"); + + EXPECT_THROW_WITH_MESSAGE(RequestInfoHeaderFormatter("UPSTREAM_METADATA([\"a\",\"b\"]", false), + EnvoyException, + "Invalid header configuration. Expected format " + "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " + "UPSTREAM_METADATA([\"a\",\"b\"]"); // Non-string elements. EXPECT_THROW_WITH_MESSAGE( - RequestInfoHeaderFormatter requestInfoHeaderFormatter("UPSTREAM_METADATA([\"a\", 1])", false), - EnvoyException, - "Incorrect header configuration. Expected format " + RequestInfoHeaderFormatter("UPSTREAM_METADATA([\"a\", 1])", false), EnvoyException, + "Invalid header configuration. Expected format " "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " "UPSTREAM_METADATA([\"a\", 1]), because JSON field from line 1 accessed with type 'String' " "does not match actual type 'Integer'."); - // Non-array parameters. + // Invalid string elements. EXPECT_THROW_WITH_MESSAGE( - RequestInfoHeaderFormatter requestInfoHeaderFormatter("UPSTREAM_METADATA({\"a\":1})", false), + RequestInfoHeaderFormatter("UPSTREAM_METADATA([\"a\", \"\\unothex\"])", false), EnvoyException, - "Incorrect header configuration. Expected format " + "Invalid header configuration. Expected format " + "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " + "UPSTREAM_METADATA([\"a\", \"\\unothex\"]), because JSON supplied is not valid. " + "Error(offset 7, line 1): Incorrect hex digit after \\u escape in string.\n"); + + // Non-array parameters. + EXPECT_THROW_WITH_MESSAGE( + RequestInfoHeaderFormatter("UPSTREAM_METADATA({\"a\":1})", false), EnvoyException, + "Invalid header configuration. Expected format " "UPSTREAM_METADATA([\"namespace\", \"k\", ...]), actual format " "UPSTREAM_METADATA({\"a\":1}), because JSON field from line 1 accessed with type 'Array' " "does not match actual type 'Object'."); } +TEST(HeaderParserTest, TestParseInternal) { + struct TestCase { + std::string input_; + Optional expected_output_; + Optional expected_exception_; + }; + + static const TestCase test_cases[] = { + // Valid inputs + {"%PROTOCOL%", {"HTTP/1.1"}, {}}, + {"[%PROTOCOL%", {"[HTTP/1.1"}, {}}, + {"%PROTOCOL%]", {"HTTP/1.1]"}, {}}, + {"[%PROTOCOL%]", {"[HTTP/1.1]"}, {}}, + {"%%%PROTOCOL%", {"%HTTP/1.1"}, {}}, + {"%PROTOCOL%%%", {"HTTP/1.1%"}, {}}, + {"%%%PROTOCOL%%%", {"%HTTP/1.1%"}, {}}, + {"%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%", {"127.0.0.1"}, {}}, + {"%UPSTREAM_METADATA([\"ns\", \"key\"])%", {"value"}, {}}, + {"[%UPSTREAM_METADATA([\"ns\", \"key\"])%", {"[value"}, {}}, + {"%UPSTREAM_METADATA([\"ns\", \"key\"])%]", {"value]"}, {}}, + {"[%UPSTREAM_METADATA([\"ns\", \"key\"])%]", {"[value]"}, {}}, + {"%UPSTREAM_METADATA([\"ns\", \t \"key\"])%", {"value"}, {}}, + {"%UPSTREAM_METADATA([\"ns\", \n \"key\"])%", {"value"}, {}}, + {"%UPSTREAM_METADATA( \t [ \t \"ns\" \t , \t \"key\" \t ] \t )%", {"value"}, {}}, + + // Unescaped % + {"%", {}, {"Invalid header configuration. Un-escaped % at position 0"}}, + {"before %", {}, {"Invalid header configuration. Un-escaped % at position 7"}}, + {"%% infix %", {}, {"Invalid header configuration. Un-escaped % at position 9"}}, + + // Unknown variable names + {"%INVALID%", {}, {"field 'INVALID' not supported as custom header"}}, + {"before %INVALID%", {}, {"field 'INVALID' not supported as custom header"}}, + {"%INVALID% after", {}, {"field 'INVALID' not supported as custom header"}}, + {"before %INVALID% after", {}, {"field 'INVALID' not supported as custom header"}}, + + // Un-terminated variable expressions. + {"%VAR", {}, {"Invalid header configuration. Un-terminated variable expression 'VAR'"}}, + {"%%%VAR", {}, {"Invalid header configuration. Un-terminated variable expression 'VAR'"}}, + {"before %VAR", + {}, + {"Invalid header configuration. Un-terminated variable expression 'VAR'"}}, + {"before %%%VAR", + {}, + {"Invalid header configuration. Un-terminated variable expression 'VAR'"}}, + {"before %VAR after", + {}, + {"Invalid header configuration. Un-terminated variable expression 'VAR after'"}}, + {"before %%%VAR after", + {}, + {"Invalid header configuration. Un-terminated variable expression 'VAR after'"}}, + {"% ", {}, {"Invalid header configuration. Un-terminated variable expression ' '"}}, + + // Un-terminated variable expressions with arguments. + {"%VAR(no array)%", + {}, + {"Invalid header configuration. Expecting JSON array of arguments after " + "'VAR(', but found 'n'"}}, + {"%VAR( no array)%", + {}, + {"Invalid header configuration. Expecting JSON array of arguments after " + "'VAR( ', but found 'n'"}}, + {"%VAR([)", + {}, + {"Invalid header configuration. Expecting '\"' or whitespace after 'VAR([', but found ')'"}}, + {"%VAR([ )", + {}, + {"Invalid header configuration. Expecting '\"' or whitespace after 'VAR([ ', but found " + "')'"}}, + {"%VAR([\"x\")%", + {}, + {"Invalid header configuration. Expecting ',', ']', or whitespace after " + "'VAR([\"x\"', but found ')'"}}, + {"%VAR([\"x\" )%", + {}, + {"Invalid header configuration. Expecting ',', ']', or whitespace after " + "'VAR([\"x\" ', but found ')'"}}, + {"%VAR([\"x\\", + {}, + {"Invalid header configuration. Un-terminated backslash in JSON string after 'VAR([\"x'"}}, + {"%VAR([\"x\"]!", + {}, + {"Invalid header configuration. Expecting ')' or whitespace after " + "'VAR([\"x\"]', but found '!'"}}, + {"%VAR([\"x\"] !", + {}, + {"Invalid header configuration. Expecting ')' or whitespace after " + "'VAR([\"x\"] ', but found '!'"}}, + {"%VAR([\"x\"])!", + {}, + {"Invalid header configuration. Expecting '%' or whitespace after " + "'VAR([\"x\"])', but found '!'"}}, + {"%VAR([\"x\"]) !", + {}, + {"Invalid header configuration. Expecting '%' or whitespace after " + "'VAR([\"x\"]) ', but found '!'"}}, + + // Argument errors + {"%VAR()%", + {}, + {"Invalid header configuration. Expecting JSON array of arguments after 'VAR(', but found " + "')'"}}, + {"%VAR( )%", + {}, + {"Invalid header configuration. Expecting JSON array of arguments after 'VAR( ', but found " + "')'"}}, + {"%VAR([])%", + {}, + {"Invalid header configuration. Expecting '\"' or whitespace after 'VAR([', but found ']'"}}, + {"%VAR( [ ] )%", + {}, + {"Invalid header configuration. Expecting '\"' or whitespace after 'VAR( [ ', but found " + "']'"}}, + {"%VAR([\"ns\",])%", + {}, + {"Invalid header configuration. Expecting '\"' or whitespace after 'VAR([\"ns\",', but " + "found ']'"}}, + {"%VAR( [ \"ns\" , ] )%", + {}, + {"Invalid header configuration. Expecting '\"' or whitespace after 'VAR( [ \"ns\" , ', but " + "found ']'"}}, + {"%VAR({\"ns\": \"key\"})%", + {}, + {"Invalid header configuration. Expecting JSON array of arguments after 'VAR(', but found " + "'{'"}}, + {"%VAR(\"ns\", \"key\")%", + {}, + {"Invalid header configuration. Expecting JSON array of arguments after 'VAR(', but found " + "'\"'"}}, + + // Invalid arguments + {"%UPSTREAM_METADATA%", + {}, + {"Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " + "...]), actual format UPSTREAM_METADATA"}}, + {"%UPSTREAM_METADATA([\"ns\"])%", + {}, + {"Invalid header configuration. Expected format UPSTREAM_METADATA([\"namespace\", \"k\", " + "...]), actual format UPSTREAM_METADATA([\"ns\"])"}}, + }; + + NiceMock request_info; + Optional protocol = Envoy::Http::Protocol::Http11; + ON_CALL(request_info, protocol()).WillByDefault(ReturnRef(protocol)); + + std::shared_ptr> host( + new NiceMock()); + ON_CALL(request_info, upstreamHost()).WillByDefault(Return(host)); + + // Metadata with percent signs in the key. + envoy::api::v2::Metadata metadata = TestUtility::parseYaml( + R"EOF( + filter_metadata: + ns: + key: value + )EOF"); + ON_CALL(*host, metadata()).WillByDefault(ReturnRef(metadata)); + + for (const auto& test_case : test_cases) { + Protobuf::RepeatedPtrField to_add; + envoy::api::v2::HeaderValueOption* header = to_add.Add(); + header->mutable_header()->set_key("x-header"); + header->mutable_header()->set_value(test_case.input_); + + if (test_case.expected_exception_.valid()) { + EXPECT_FALSE(test_case.expected_output_.valid()); + EXPECT_THROW_WITH_MESSAGE(HeaderParser::configure(to_add), EnvoyException, + test_case.expected_exception_.value()); + continue; + } + + HeaderParserPtr req_header_parser = HeaderParser::configure(to_add); + + Http::TestHeaderMapImpl headerMap{{":method", "POST"}}; + req_header_parser->evaluateHeaders(headerMap, request_info); + + std::string descriptor = fmt::format("for test case input: {}", test_case.input_); + + EXPECT_TRUE(headerMap.has("x-header")) << descriptor; + EXPECT_TRUE(test_case.expected_output_.valid()) << descriptor; + EXPECT_EQ(test_case.expected_output_.value(), headerMap.get_("x-header")) << descriptor; + } +} + TEST(HeaderParserTest, EvaluateHeaders) { const std::string json = R"EOF( { @@ -329,8 +433,8 @@ TEST(HeaderParserTest, EvaluateHeaders) { ] } )EOF"; - HeaderParserPtr req_header_parser = Envoy::Router::HeaderParser::configure( - parseRouteFromJson(json).route().request_headers_to_add()); + HeaderParserPtr req_header_parser = + HeaderParser::configure(parseRouteFromJson(json).route().request_headers_to_add()); Http::TestHeaderMapImpl headerMap{{":method", "POST"}}; NiceMock request_info; req_header_parser->evaluateHeaders(headerMap, request_info); @@ -351,8 +455,8 @@ TEST(HeaderParserTest, EvaluateEmptyHeaders) { ] } )EOF"; - HeaderParserPtr req_header_parser = Envoy::Router::HeaderParser::configure( - parseRouteFromJson(json).route().request_headers_to_add()); + HeaderParserPtr req_header_parser = + HeaderParser::configure(parseRouteFromJson(json).route().request_headers_to_add()); Http::TestHeaderMapImpl headerMap{{":method", "POST"}}; std::shared_ptr> host( new NiceMock()); @@ -378,8 +482,8 @@ TEST(HeaderParserTest, EvaluateStaticHeaders) { ] } )EOF"; - HeaderParserPtr req_header_parser = Envoy::Router::HeaderParser::configure( - parseRouteFromJson(json).route().request_headers_to_add()); + HeaderParserPtr req_header_parser = + HeaderParser::configure(parseRouteFromJson(json).route().request_headers_to_add()); Http::TestHeaderMapImpl headerMap{{":method", "POST"}}; NiceMock request_info; req_header_parser->evaluateHeaders(headerMap, request_info); @@ -387,6 +491,85 @@ TEST(HeaderParserTest, EvaluateStaticHeaders) { EXPECT_EQ("static-value", headerMap.get_("static-header")); } +TEST(HeaderParserTest, EvaluateCompoundHeaders) { + const std::string yaml = R"EOF( +match: { prefix: "/new_endpoint" } +route: + cluster: www2 + request_headers_to_add: + - header: + key: "x-prefix" + value: "prefix-%CLIENT_IP%" + - header: + key: "x-suffix" + value: "%CLIENT_IP%-suffix" + - header: + key: "x-both" + value: "prefix-%CLIENT_IP%-suffix" + - header: + key: "x-escaping-1" + value: "%%%CLIENT_IP%%%" + - header: + key: "x-escaping-2" + value: "%%%%%%" + - header: + key: "x-multi" + value: "%PROTOCOL% from %CLIENT_IP%" + - header: + key: "x-multi-back-to-back" + value: "%PROTOCOL%%CLIENT_IP%" + - header: + key: "x-metadata" + value: "%UPSTREAM_METADATA([\"namespace\", \"%key%\"])%" + )EOF"; + + HeaderParserPtr req_header_parser = + HeaderParser::configure(parseRouteFromV2Yaml(yaml).route().request_headers_to_add()); + Http::TestHeaderMapImpl headerMap{{":method", "POST"}}; + NiceMock request_info; + Optional protocol = Envoy::Http::Protocol::Http11; + ON_CALL(request_info, protocol()).WillByDefault(ReturnRef(protocol)); + + std::shared_ptr> host( + new NiceMock()); + ON_CALL(request_info, upstreamHost()).WillByDefault(Return(host)); + + // Metadata with percent signs in the key. + envoy::api::v2::Metadata metadata = TestUtility::parseYaml( + R"EOF( + filter_metadata: + namespace: + "%key%": value + )EOF"); + ON_CALL(*host, metadata()).WillByDefault(ReturnRef(metadata)); + + req_header_parser->evaluateHeaders(headerMap, request_info); + + EXPECT_TRUE(headerMap.has("x-prefix")); + EXPECT_EQ("prefix-127.0.0.1", headerMap.get_("x-prefix")); + + EXPECT_TRUE(headerMap.has("x-suffix")); + EXPECT_EQ("127.0.0.1-suffix", headerMap.get_("x-suffix")); + + EXPECT_TRUE(headerMap.has("x-both")); + EXPECT_EQ("prefix-127.0.0.1-suffix", headerMap.get_("x-both")); + + EXPECT_TRUE(headerMap.has("x-escaping-1")); + EXPECT_EQ("%127.0.0.1%", headerMap.get_("x-escaping-1")); + + EXPECT_TRUE(headerMap.has("x-escaping-2")); + EXPECT_EQ("%%%", headerMap.get_("x-escaping-2")); + + EXPECT_TRUE(headerMap.has("x-multi")); + EXPECT_EQ("HTTP/1.1 from 127.0.0.1", headerMap.get_("x-multi")); + + EXPECT_TRUE(headerMap.has("x-multi-back-to-back")); + EXPECT_EQ("HTTP/1.1127.0.0.1", headerMap.get_("x-multi-back-to-back")); + + EXPECT_TRUE(headerMap.has("x-metadata")); + EXPECT_EQ("value", headerMap.get_("x-metadata")); +} + TEST(HeaderParserTest, EvaluateHeadersWithAppendFalse) { const std::string json = R"EOF( { @@ -457,8 +640,8 @@ match: { prefix: "/new_endpoint" } )EOF"; const auto route = parseRouteFromV2Yaml(yaml).route(); - HeaderParserPtr resp_header_parser = Envoy::Router::HeaderParser::configure( - route.response_headers_to_add(), route.response_headers_to_remove()); + HeaderParserPtr resp_header_parser = + HeaderParser::configure(route.response_headers_to_add(), route.response_headers_to_remove()); Http::TestHeaderMapImpl headerMap{{":method", "POST"}, {"x-safe", "safe"}, {"x-nope", "nope"}}; NiceMock request_info; resp_header_parser->evaluateHeaders(headerMap, request_info);