diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index 9a00ac1a29d93..6497767fde45e 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -163,6 +163,13 @@ The following command operators are supported: In typed JSON logs, START_TIME is always rendered as a string. +%REQUEST_HEADERS_BYTES% + HTTP + Uncompressed bytes of request headers. + + TCP + Not implemented (0). + %BYTES_RECEIVED% HTTP Body bytes received. @@ -213,6 +220,20 @@ The following command operators are supported: Connection termination details may provide additional information about why the connection was terminated by Envoy for L4 reasons. +%RESPONSE_HEADERS_BYTES% + HTTP + Uncompressed bytes of response headers. + + TCP + Not implemented (0). + +%RESPONSE_TRAILERS_BYTES% + HTTP + Uncompressed bytes of response trailers. + + TCP + Not implemented (0). + %BYTES_SENT% HTTP Body bytes sent. For WebSocket connection it will also include response header bytes. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index d899e6053f255..0841fde42af67 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -32,6 +32,7 @@ Removed Config or Runtime New Features ------------ * access log: added the :ref:`formatters ` extension point for custom formatters (command operators). +* access log: support command operator: %REQUEST_HEADERS_BYTES%, %RESPONSE_HEADERS_BYTES% and %RESPONSE_TRAILERS_BYTES%. * http: added support for :ref:`:ref:`preconnecting `. Preconnecting is off by default, but recommended for clusters serving latency-sensitive traffic, especially if using HTTP/1.1. * tcp_proxy: add support for converting raw TCP streams into HTTP/1.1 CONNECT requests. See :ref:`upgrade documentation ` for details. diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index d7fca3cf7a493..de491a4885584 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -357,6 +357,15 @@ FormatterProviderPtr SubstitutionFormatParser::parseBuiltinCommand(const std::st return std::make_unique(token); } else if (absl::StartsWith(token, "GRPC_STATUS")) { return std::make_unique("grpc-status", "", absl::optional()); + } else if (absl::StartsWith(token, "REQUEST_HEADERS_BYTES")) { + return std::make_unique( + HeadersByteSizeFormatter::HeaderType::RequestHeaders); + } else if (absl::StartsWith(token, "RESPONSE_HEADERS_BYTES")) { + return std::make_unique( + HeadersByteSizeFormatter::HeaderType::ResponseHeaders); + } else if (absl::StartsWith(token, "RESPONSE_TRAILERS_BYTES")) { + return std::make_unique( + HeadersByteSizeFormatter::HeaderType::ResponseTrailers); } return nullptr; @@ -955,6 +964,41 @@ ResponseTrailerFormatter::formatValue(const Http::RequestHeaderMap&, const Http: return HeaderFormatter::formatValue(response_trailers); } +HeadersByteSizeFormatter::HeadersByteSizeFormatter(const HeaderType header_type) + : header_type_(header_type) {} + +uint64_t HeadersByteSizeFormatter::extractHeadersByteSize( + const Http::RequestHeaderMap& request_headers, const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers) const { + switch (header_type_) { + case HeaderType::RequestHeaders: + return request_headers.byteSize(); + case HeaderType::ResponseHeaders: + return response_headers.byteSize(); + case HeaderType::ResponseTrailers: + return response_trailers.byteSize(); + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +absl::optional +HeadersByteSizeFormatter::format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo&, absl::string_view) const { + return absl::StrCat(extractHeadersByteSize(request_headers, response_headers, response_trailers)); +} + +ProtobufWkt::Value +HeadersByteSizeFormatter::formatValue(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo&, absl::string_view) const { + return ValueUtil::numberValue( + extractHeadersByteSize(request_headers, response_headers, response_trailers)); +} + GrpcStatusFormatter::GrpcStatusFormatter(const std::string& main_header, const std::string& alternative_header, absl::optional max_length) diff --git a/source/common/formatter/substitution_formatter.h b/source/common/formatter/substitution_formatter.h index 2d1f121c4872e..4d53c03374f6f 100644 --- a/source/common/formatter/substitution_formatter.h +++ b/source/common/formatter/substitution_formatter.h @@ -228,6 +228,33 @@ class HeaderFormatter { absl::optional max_length_; }; +/** + * FormatterProvider for headers byte size. + */ +class HeadersByteSizeFormatter : public FormatterProvider { +public: + // TODO(taoxuy): Add RequestTrailers here. + enum class HeaderType { RequestHeaders, ResponseHeaders, ResponseTrailers }; + + HeadersByteSizeFormatter(const HeaderType header_type); + + absl::optional format(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo&, + absl::string_view) const override; + ProtobufWkt::Value formatValue(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers, + const StreamInfo::StreamInfo&, absl::string_view) const override; + +private: + uint64_t extractHeadersByteSize(const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers) const; + HeaderType header_type_; +}; + /** * FormatterProvider for request headers. */ diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index 1e6eb05c0cea8..ce4d3909af73d 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -141,6 +141,40 @@ name: accesslog output_); } +TEST_F(AccessLogImplTest, HeadersBytes) { + const std::string yaml = R"EOF( +name: accesslog +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + log_format: + text_format_source: + inline_string: "%REQUEST_HEADERS_BYTES% %RESPONSE_HEADERS_BYTES% %RESPONSE_TRAILERS_BYTES%" + )EOF"; + + InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); + + EXPECT_CALL(*file_, write(_)); + request_headers_.addCopy("request_header_key", "request_header_val"); + response_headers_.addCopy("response_header_key", "response_header_val"); + response_trailers_.addCopy("response_trailer_key", "response_trailer_val"); + + // request headers: + // :method: GET + // :path: / + // request_header_key: request_header_val + // + // response headers: + // response_header_key: response_header_val + // + // response trailers: + // response_trailer_key: response_trailer_val + + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + + EXPECT_EQ(output_, "52 38 40"); +} + TEST_F(AccessLogImplTest, EnvoyUpstreamServiceTime) { const std::string yaml = R"EOF( name: accesslog diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 87d81342169be..75e9a2c8abdb4 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -1050,6 +1050,42 @@ TEST(SubstitutionFormatterTest, requestHeaderFormatter) { } } +TEST(SubstitutionFormatterTest, headersByteSizeFormatter) { + StreamInfo::MockStreamInfo stream_info; + Http::TestRequestHeaderMapImpl request_header{{":method", "GET"}, {":path", "/"}}; + Http::TestResponseHeaderMapImpl response_header{{":method", "PUT"}}; + Http::TestResponseTrailerMapImpl response_trailer{{":method", "POST"}, {"test-2", "test-2"}}; + std::string body; + + { + HeadersByteSizeFormatter formatter(HeadersByteSizeFormatter::HeaderType::RequestHeaders); + EXPECT_EQ( + formatter.format(request_header, response_header, response_trailer, stream_info, body), + "16"); + EXPECT_THAT( + formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), + ProtoEq(ValueUtil::numberValue(16))); + } + { + HeadersByteSizeFormatter formatter(HeadersByteSizeFormatter::HeaderType::ResponseHeaders); + EXPECT_EQ( + formatter.format(request_header, response_header, response_trailer, stream_info, body), + "10"); + EXPECT_THAT( + formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), + ProtoEq(ValueUtil::numberValue(10))); + } + { + HeadersByteSizeFormatter formatter(HeadersByteSizeFormatter::HeaderType::ResponseTrailers); + EXPECT_EQ( + formatter.format(request_header, response_header, response_trailer, stream_info, body), + "23"); + EXPECT_THAT( + formatter.formatValue(request_header, response_header, response_trailer, stream_info, body), + ProtoEq(ValueUtil::numberValue(23))); + } +} + TEST(SubstitutionFormatterTest, responseHeaderFormatter) { StreamInfo::MockStreamInfo stream_info; Http::TestRequestHeaderMapImpl request_header{{":method", "GET"}, {":path", "/"}}; diff --git a/test/common/router/router_upstream_log_test.cc b/test/common/router/router_upstream_log_test.cc index 7821291c80a55..3a7508903c819 100644 --- a/test/common/router/router_upstream_log_test.cc +++ b/test/common/router/router_upstream_log_test.cc @@ -326,5 +326,43 @@ name: accesslog EXPECT_LE(std::abs(std::difftime(log_time, now)), 300); } +// Test request headers/response headers/response trailers byte size. +TEST_F(RouterUpstreamLogTest, HeaderByteSize) { + const std::string yaml = R"EOF( +name: accesslog +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + log_format: + text_format_source: + inline_string: "%REQUEST_HEADERS_BYTES% %RESPONSE_HEADERS_BYTES% %RESPONSE_TRAILERS_BYTES%" + path: "/dev/null" + )EOF"; + + envoy::config::accesslog::v3::AccessLog upstream_log; + TestUtility::loadFromYaml(yaml, upstream_log); + + init(absl::optional(upstream_log)); + run(200, {{"request-header-name", "request-header-val"}}, + {{"response-header-name", "response-header-val"}}, + {{"response-trailer-name", "response-trailer-val"}}); + + EXPECT_EQ(output_.size(), 1U); + // Request headers: + // scheme: http + // :method: GET + // :authority: host + // :path: / + // x-envoy-expected-rq-timeout-ms: 10 + // request-header-name: request-header-val + + // Response headers: + // :status: 200 + // response-header-name: response-header-val + + // Response trailers: + // response-trailer-name: response-trailer-val + EXPECT_EQ(output_.front(), "110 49 41"); +} + } // namespace Router } // namespace Envoy