diff --git a/docs/root/configuration/access_log.rst b/docs/root/configuration/access_log.rst index 9350cc5fb5227..e65d4fae2559e 100644 --- a/docs/root/configuration/access_log.rst +++ b/docs/root/configuration/access_log.rst @@ -296,6 +296,19 @@ The following command operators are supported: TCP Not implemented ("-"). +%REQ_WITHOUT_QUERY(X?Y):Z% + HTTP + Same as **%REQ(X?Y):Z%** but it removes query string from a header entry value. + For example, for the following header entry: + + ``":path" : "/?ok=true"`` + + * ``%REQ(:PATH)%`` will log: ``/?ok=true`` + * ``%REQ_WITHOUT_QUERY(:PATH)%`` will log: ``/`` + + TCP + Not implemented ("-"). + %RESP(X?Y):Z% HTTP Same as **%REQ(X?Y):Z%** but taken from HTTP response headers. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 484bc9260060c..b64dd78df4bee 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -4,6 +4,7 @@ Version history 1.12.0 (pending) ================ * access log: added :ref:`buffering ` and :ref:`periodical flushing ` support to gRPC access logger. Defaults to 16KB buffer and flushing every 1 second. +* access log: added %REQ_WITHOUT_QUERY(X?Y):Z formatter to remove query string from a header entry value. * admin: added ability to configure listener :ref:`socket options `. * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. * config: added access log :ref:`extension filter`. diff --git a/source/common/access_log/access_log_formatter.cc b/source/common/access_log/access_log_formatter.cc index 899ae27ff8d1d..450fa14af61aa 100644 --- a/source/common/access_log/access_log_formatter.cc +++ b/source/common/access_log/access_log_formatter.cc @@ -222,7 +222,6 @@ void AccessLogFormatParser::parseCommand(const std::string& token, const size_t std::vector AccessLogFormatParser::parse(const std::string& format) { std::string current_token; std::vector formatters; - const std::string DYNAMIC_META_TOKEN = "DYNAMIC_METADATA("; const std::regex command_w_args_regex(R"EOF(%([A-Z]|_)+(\([^\)]*\))?(:[0-9]+)?(%))EOF"); for (size_t pos = 0; pos < format.length(); ++pos) { @@ -245,14 +244,16 @@ std::vector AccessLogFormatParser::parse(const std::string pos += 1; int command_end_position = pos + token.length(); - if (absl::StartsWith(token, "REQ(")) { + const bool req_without_query = absl::StartsWith(token, "REQ_WITHOUT_QUERY("); + if (absl::StartsWith(token, "REQ(") || req_without_query) { std::string main_header, alternative_header; absl::optional max_length; - parseCommandHeader(token, ReqParamStart, main_header, alternative_header, max_length); + parseCommandHeader(token, req_without_query ? ReqWithoutQueryParamStart : ReqParamStart, + main_header, alternative_header, max_length); - formatters.emplace_back(FormatterProviderPtr{ - new RequestHeaderFormatter(main_header, alternative_header, max_length)}); + formatters.emplace_back(FormatterProviderPtr{new RequestHeaderFormatter( + main_header, alternative_header, max_length, req_without_query)}); } else if (absl::StartsWith(token, "RESP(")) { std::string main_header, alternative_header; absl::optional max_length; @@ -269,13 +270,13 @@ std::vector AccessLogFormatParser::parse(const std::string formatters.emplace_back(FormatterProviderPtr{ new ResponseTrailerFormatter(main_header, alternative_header, max_length)}); - } else if (absl::StartsWith(token, DYNAMIC_META_TOKEN)) { + } else if (absl::StartsWith(token, "DYNAMIC_METADATA(")) { std::string filter_namespace; absl::optional max_length; std::vector path; - const size_t start = DYNAMIC_META_TOKEN.size(); - parseCommand(token, start, ":", filter_namespace, path, max_length); + parseCommand(token, DynamicMetadataParamStart, ":", filter_namespace, path, max_length); + formatters.emplace_back( FormatterProviderPtr{new DynamicMetadataFormatter(filter_namespace, path, max_length)}); } else if (absl::StartsWith(token, "START_TIME")) { @@ -508,8 +509,9 @@ std::string PlainStringFormatter::format(const Http::HeaderMap&, const Http::Hea HeaderFormatter::HeaderFormatter(const std::string& main_header, const std::string& alternative_header, - absl::optional max_length) - : main_header_(main_header), alternative_header_(alternative_header), max_length_(max_length) {} + absl::optional max_length, bool remove_query) + : main_header_(main_header), alternative_header_(alternative_header), max_length_(max_length), + remove_query_(remove_query) {} std::string HeaderFormatter::format(const Http::HeaderMap& headers) const { const Http::HeaderEntry* header = headers.get(main_header_); @@ -522,7 +524,14 @@ std::string HeaderFormatter::format(const Http::HeaderMap& headers) const { if (!header) { header_value_string = UnspecifiedValueString; } else { - header_value_string = std::string(header->value().getStringView()); + absl::string_view header_value = header->value().getStringView(); + if (remove_query_) { + const size_t query_pos = header_value.find('?'); + header_value_string = std::string( + header_value.data(), query_pos != header_value.npos ? query_pos : header_value.size()); + } else { + header_value_string = std::string(header_value); + } } if (max_length_ && header_value_string.length() > max_length_.value()) { @@ -535,7 +544,7 @@ std::string HeaderFormatter::format(const Http::HeaderMap& headers) const { ResponseHeaderFormatter::ResponseHeaderFormatter(const std::string& main_header, const std::string& alternative_header, absl::optional max_length) - : HeaderFormatter(main_header, alternative_header, max_length) {} + : HeaderFormatter(main_header, alternative_header, max_length, false) {} std::string ResponseHeaderFormatter::format(const Http::HeaderMap&, const Http::HeaderMap& response_headers, @@ -546,8 +555,8 @@ std::string ResponseHeaderFormatter::format(const Http::HeaderMap&, RequestHeaderFormatter::RequestHeaderFormatter(const std::string& main_header, const std::string& alternative_header, - absl::optional max_length) - : HeaderFormatter(main_header, alternative_header, max_length) {} + absl::optional max_length, bool remove_query) + : HeaderFormatter(main_header, alternative_header, max_length, remove_query) {} std::string RequestHeaderFormatter::format(const Http::HeaderMap& request_headers, const Http::HeaderMap&, const Http::HeaderMap&, @@ -558,7 +567,7 @@ std::string RequestHeaderFormatter::format(const Http::HeaderMap& request_header ResponseTrailerFormatter::ResponseTrailerFormatter(const std::string& main_header, const std::string& alternative_header, absl::optional max_length) - : HeaderFormatter(main_header, alternative_header, max_length) {} + : HeaderFormatter(main_header, alternative_header, max_length, false) {} std::string ResponseTrailerFormatter::format(const Http::HeaderMap&, const Http::HeaderMap&, const Http::HeaderMap& response_trailers, diff --git a/source/common/access_log/access_log_formatter.h b/source/common/access_log/access_log_formatter.h index 103657dcb40db..ec10f10cf91e5 100644 --- a/source/common/access_log/access_log_formatter.h +++ b/source/common/access_log/access_log_formatter.h @@ -61,8 +61,10 @@ class AccessLogFormatParser { // the indexes of where the parameters for each directive is expected to begin static const size_t ReqParamStart{sizeof("REQ(") - 1}; + static const size_t ReqWithoutQueryParamStart{sizeof("REQ_WITHOUT_QUERY(") - 1}; static const size_t RespParamStart{sizeof("RESP(") - 1}; static const size_t TrailParamStart{sizeof("TRAILER(") - 1}; + static const size_t DynamicMetadataParamStart{sizeof("DYNAMIC_METADATA(") - 1}; static const size_t StartTimeParamStart{sizeof("START_TIME(") - 1}; }; @@ -137,7 +139,7 @@ class PlainStringFormatter : public FormatterProvider { class HeaderFormatter { public: HeaderFormatter(const std::string& main_header, const std::string& alternative_header, - absl::optional max_length); + absl::optional max_length, bool remove_query); std::string format(const Http::HeaderMap& headers) const; @@ -145,6 +147,7 @@ class HeaderFormatter { Http::LowerCaseString main_header_; Http::LowerCaseString alternative_header_; absl::optional max_length_; + const bool remove_query_; }; /** @@ -153,7 +156,7 @@ class HeaderFormatter { class RequestHeaderFormatter : public FormatterProvider, HeaderFormatter { public: RequestHeaderFormatter(const std::string& main_header, const std::string& alternative_header, - absl::optional max_length); + absl::optional max_length, bool remove_query); // Formatter::format std::string format(const Http::HeaderMap& request_headers, const Http::HeaderMap&, diff --git a/test/common/access_log/access_log_formatter_test.cc b/test/common/access_log/access_log_formatter_test.cc index d10665a034f47..61fdfe267cc51 100644 --- a/test/common/access_log/access_log_formatter_test.cc +++ b/test/common/access_log/access_log_formatter_test.cc @@ -546,30 +546,37 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { TEST(AccessLogFormatterTest, requestHeaderFormatter) { StreamInfo::MockStreamInfo stream_info; - Http::TestHeaderMapImpl request_header{{":method", "GET"}, {":path", "/"}}; + Http::TestHeaderMapImpl request_header{{":method", "GET"}, {":path", "/?ok=true"}}; Http::TestHeaderMapImpl response_header{{":method", "PUT"}}; Http::TestHeaderMapImpl response_trailer{{":method", "POST"}, {"test-2", "test-2"}}; { - RequestHeaderFormatter formatter(":Method", "", absl::optional()); + RequestHeaderFormatter formatter(":Method", "", absl::optional(), false); EXPECT_EQ("GET", formatter.format(request_header, response_header, response_trailer, stream_info)); } { - RequestHeaderFormatter formatter(":path", ":method", absl::optional()); + RequestHeaderFormatter formatter(":path", ":method", absl::optional(), false); + EXPECT_EQ("/?ok=true", + formatter.format(request_header, response_header, response_trailer, stream_info)); + } + + { + RequestHeaderFormatter formatter(":path", ":method", absl::optional(), + true /* to remove query string from :path */); EXPECT_EQ("/", formatter.format(request_header, response_header, response_trailer, stream_info)); } { - RequestHeaderFormatter formatter(":TEST", ":METHOD", absl::optional()); + RequestHeaderFormatter formatter(":TEST", ":METHOD", absl::optional(), false); EXPECT_EQ("GET", formatter.format(request_header, response_header, response_trailer, stream_info)); } { - RequestHeaderFormatter formatter("does_not_exist", "", absl::optional()); + RequestHeaderFormatter formatter("does_not_exist", "", absl::optional(), false); EXPECT_EQ("-", formatter.format(request_header, response_header, response_trailer, stream_info)); } @@ -920,7 +927,7 @@ TEST(AccessLogFormatterTest, JsonFormatterMultiTokenTest) { TEST(AccessLogFormatterTest, CompositeFormatterSuccess) { StreamInfo::MockStreamInfo stream_info; - Http::TestHeaderMapImpl request_header{{"first", "GET"}, {":path", "/"}}; + Http::TestHeaderMapImpl request_header{{"first", "GET"}, {":path", "/?ok=true"}}; Http::TestHeaderMapImpl response_header{{"second", "PUT"}, {"test", "test"}}; Http::TestHeaderMapImpl response_trailer{{"third", "POST"}, {"test-2", "test-2"}}; @@ -1028,6 +1035,20 @@ TEST(AccessLogFormatterTest, CompositeFormatterSuccess) { EXPECT_EQ("%%|%%123456000|1522796769%%123|1%%1522796769", formatter.format(request_header, response_header, response_trailer, stream_info)); } + + { + const std::string format = "%REQ(:PATH)%"; + FormatterImpl formatter(format); + EXPECT_EQ("/?ok=true", + formatter.format(request_header, response_header, response_trailer, stream_info)); + } + + { + const std::string format = "%REQ_WITHOUT_QUERY(:PATH)%"; + FormatterImpl formatter(format); + EXPECT_EQ("/", + formatter.format(request_header, response_header, response_trailer, stream_info)); + } } TEST(AccessLogFormatterTest, ParserFailures) {