formatter: print request header without query string

CODEOWNERS

@@ -190,3 +190,5 @@ extensions/filters/http/oauth2 @rgs1 @derekargueta @snowp
 /*/extensions/http/original_ip_detection/xff @rgs1 @alyssawilk @antoniovicente
 # set_metadata extension
 /*/extensions/filters/http/set_metadata @aguinet @snowp
+# Formatters
+/*/extensions/formatter/req_without_query @dio @tsaarni

api/BUILD

@@ -245,6 +245,7 @@ proto_library(
         "//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg",
         "//envoy/extensions/filters/udp/dns_filter/v3alpha:pkg",
         "//envoy/extensions/filters/udp/udp_proxy/v3:pkg",
+        "//envoy/extensions/formatter/req_without_query/v3:pkg",
         "//envoy/extensions/health_checkers/redis/v3:pkg",
         "//envoy/extensions/http/header_formatters/preserve_case/v3:pkg",
         "//envoy/extensions/http/original_ip_detection/custom_header/v3:pkg",

api/envoy/config/core/v3/substitution_format_string.proto

@@ -109,5 +109,6 @@ message SubstitutionFormatString {
 
   // Specifies a collection of Formatter plugins that can be called from the access log configuration.
   // See the formatters extensions documentation for details.
+  // [#extension-category: envoy.formatter]
   repeated TypedExtensionConfig formatters = 6;
 }

api/envoy/extensions/formatter/req_without_query/v3/BUILD

@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
+)

api/envoy/extensions/formatter/req_without_query/v3/req_without_query.proto

@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+package envoy.extensions.formatter.req_without_query.v3;
+
+import "udpa/annotations/status.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.formatter.req_without_query.v3";
+option java_outer_classname = "ReqWithoutQueryProto";
+option java_multiple_files = true;
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Formatter extension for printing request without query string]
+// [#extension: envoy.formatter.req_without_query]
+
+// ReqWithoutQuery formatter extension implements REQ_WITHOUT_QUERY command operator that
+// works the same way as :ref:`REQ <config_access_log_format_req>` except that it will
+// remove the query string. It is used to avoid logging any sensitive information into
+// the access log.
+// See :ref:`here <config_access_log>` for more information on access log configuration.
+
+// %REQ_WITHOUT_QUERY(X?Y):Z%
+//   An HTTP request header where X is the main HTTP header, Y is the alternative one, and Z is an
+//   optional parameter denoting string truncation up to Z characters long. The value is taken from
+//   the HTTP request header named X first and if it's not set, then request header Y is used. If
+//   none of the headers are present '-' symbol will be in the log.
+
+// Configuration for the request without query formatter.
+message ReqWithoutQuery {
+}

api/versioning/BUILD

@@ -128,6 +128,7 @@ proto_library(
         "//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg",
         "//envoy/extensions/filters/udp/dns_filter/v3alpha:pkg",
         "//envoy/extensions/filters/udp/udp_proxy/v3:pkg",
+        "//envoy/extensions/formatter/req_without_query/v3:pkg",
         "//envoy/extensions/health_checkers/redis/v3:pkg",
         "//envoy/extensions/http/header_formatters/preserve_case/v3:pkg",
         "//envoy/extensions/http/original_ip_detection/custom_header/v3:pkg",

docs/root/api-v3/config/config.rst

@@ -28,3 +28,4 @@ Extensions
   request_id/request_id
   http/header_formatters
   http/original_ip_detection
+  formatter/formatter

docs/root/api-v3/config/formatter/formatter.rst

@@ -0,0 +1,10 @@
+.. _api-v3_config_accesslog_formatters:
+
+Access log formatters
+=====================
+
+.. toctree::
+  :glob:
+  :maxdepth: 2
+
+  ../../extensions/formatter/*/v3/*

docs/root/configuration/observability/access_log/usage.rst

@@ -416,6 +416,8 @@ The following command operators are supported:
 %DOWNSTREAM_LOCAL_PORT%
     Similar to **%DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT%**, but only extracts the port portion of the **%DOWNSTREAM_LOCAL_ADDRESS%**
 
+.. _config_access_log_format_req:
+
 %REQ(X?Y):Z%
   HTTP
     An HTTP request header where X is the main HTTP header, Y is the alternative one, and Z is an

docs/root/version_history/current.rst

@@ -92,6 +92,7 @@ New Features
 * metric service: added support for sending metric tags as labels. This can be enabled by setting the :ref:`emit_tags_as_labels <envoy_v3_api_field_config.metrics.v3.MetricsServiceConfig.emit_tags_as_labels>` field to true.
 * proxy protocol: added support for generating the header while using the :ref:`HTTP connection manager <config_http_conn_man>`. This is done using the using the :ref:`Proxy Protocol Transport Socket <extension_envoy.transport_sockets.upstream_proxy_protocol>` on upstream clusters.
   This feature is currently affected by a memory leak `issue <https://github.com/envoyproxy/envoy/issues/16682>`_.
+* req_without_query: added access log formatter extension implementing command operator :ref:`REQ_WITHOUT_QUERY <envoy_v3_api_msg_extensions.formatter.req_without_query.v3.ReqWithoutQuery>` to log the request path, while excluding the query string.
 * tcp: added support for :ref:`preconnecting <v1.18.0:envoy_v3_api_msg_config.cluster.v3.Cluster.PreconnectPolicy>`. Preconnecting is off by default, but recommended for clusters serving latency-sensitive traffic.
 * thrift_proxy: added per upstream metrics within the :ref:`thrift router <envoy_v3_api_msg_extensions.filters.network.thrift_proxy.router.v3.Router>` for request and response size histograms.
 * tls: allow dual ECDSA/RSA certs via SDS. Previously, SDS only supported a single certificate per context, and dual cert was only supported via non-SDS.

generated_api_shadow/BUILD

@@ -245,6 +245,7 @@ proto_library(
         "//envoy/extensions/filters/network/zookeeper_proxy/v3:pkg",
         "//envoy/extensions/filters/udp/dns_filter/v3alpha:pkg",
         "//envoy/extensions/filters/udp/udp_proxy/v3:pkg",
+        "//envoy/extensions/formatter/req_without_query/v3:pkg",
         "//envoy/extensions/health_checkers/redis/v3:pkg",
         "//envoy/extensions/http/header_formatters/preserve_case/v3:pkg",
         "//envoy/extensions/http/original_ip_detection/custom_header/v3:pkg",

source/common/http/utility.cc

@@ -421,6 +421,13 @@ absl::string_view Utility::findQueryStringStart(const HeaderString& path) {
   return path_str;
 }
 
+std::string Utility::stripQueryString(const HeaderString& path) {
+  absl::string_view path_str = path.getStringView();
+  size_t query_offset = path_str.find('?');
+  return std::string(path_str.data(),
+                     query_offset != path_str.npos ? query_offset : path_str.size());
+}
+
 std::string Utility::parseCookieValue(const HeaderMap& headers, const std::string& key) {
   return parseCookie(headers, key, Http::Headers::get().Cookie.get());
 }

source/common/http/utility.h

@@ -240,6 +240,13 @@ QueryParams parseParameters(absl::string_view data, size_t start, bool decode_pa
  */
 absl::string_view findQueryStringStart(const HeaderString& path);
 
+/**
+ * Returns the path without the query string.
+ * @param path supplies a HeaderString& possibly containing a query string.
+ * @return std::string the path without query string.
+ */
+std::string stripQueryString(const HeaderString& path);
+
 /**
  * Parse a particular value out of a cookie
  * @param headers supplies the headers to get the cookie from.

source/extensions/extensions_build_config.bzl

@@ -280,6 +280,11 @@ EXTENSIONS = {
     "envoy.http.original_ip_detection.custom_header":        "//source/extensions/http/original_ip_detection/custom_header:config",
     "envoy.http.original_ip_detection.xff":                  "//source/extensions/http/original_ip_detection/xff:config",
 
+    #
+    # Formatter
+    #
+
+    "envoy.formatter.req_without_query":                "//source/extensions/formatter/req_without_query:config",
 }
 
 # These can be changed to ["//visibility:public"], for  downstream builds which

source/extensions/extensions_metadata.yaml

@@ -428,6 +428,11 @@ envoy.filters.udp_listener.udp_proxy:
   - envoy.filters.udp_listener
   security_posture: robust_to_untrusted_downstream
   status: stable
+envoy.formatter.req_without_query:
+  categories:
+  - envoy.formatter
+  security_posture: robust_to_untrusted_downstream_and_upstream
+  status: alpha
 envoy.grpc_credentials.aws_iam:
   categories:
   - envoy.grpc_credentials

source/extensions/formatter/req_without_query/BUILD

@@ -0,0 +1,34 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_extension",
+    "envoy_cc_library",
+    "envoy_extension_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_extension_package()
+
+# Access log formatter that strips query string from request path
+# Public docs: docs/root/TODO(tsaarni)
+
+envoy_cc_library(
+    name = "req_without_query_lib",
+    srcs = ["req_without_query.cc"],
+    hdrs = ["req_without_query.h"],
+    deps = [
+        "//source/common/formatter:substitution_formatter_lib",
+        "//source/common/protobuf:utility_lib",
+    ],
+)
+
+envoy_cc_extension(
+    name = "config",
+    srcs = ["config.cc"],
+    hdrs = ["config.h"],
+    deps = [
+        "//envoy/registry",
+        "//source/extensions/formatter/req_without_query:req_without_query_lib",
+        "@envoy_api//envoy/extensions/formatter/req_without_query/v3:pkg_cc_proto",
+    ],
+)

source/extensions/formatter/req_without_query/config.cc

@@ -0,0 +1,26 @@
+#include "source/extensions/formatter/req_without_query/config.h"
+
+#include "envoy/extensions/formatter/req_without_query/v3/req_without_query.pb.h"
+
+#include "source/extensions/formatter/req_without_query/req_without_query.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Formatter {
+
+::Envoy::Formatter::CommandParserPtr
+ReqWithoutQueryFactory::createCommandParserFromProto(const Protobuf::Message&) {
+  return std::make_unique<ReqWithoutQueryCommandParser>();
+}
+
+ProtobufTypes::MessagePtr ReqWithoutQueryFactory::createEmptyConfigProto() {
+  return std::make_unique<envoy::extensions::formatter::req_without_query::v3::ReqWithoutQuery>();
+}
+
+std::string ReqWithoutQueryFactory::name() const { return "envoy.formatter.req_without_query"; }
+
+REGISTER_FACTORY(ReqWithoutQueryFactory, ReqWithoutQueryFactory::CommandParserFactory);
+
+} // namespace Formatter
+} // namespace Extensions
+} // namespace Envoy

source/extensions/formatter/req_without_query/config.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "source/common/formatter/substitution_formatter.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Formatter {
+
+class ReqWithoutQueryFactory : public ::Envoy::Formatter::CommandParserFactory {
+public:
+  ::Envoy::Formatter::CommandParserPtr
+  createCommandParserFromProto(const Protobuf::Message&) override;
+  ProtobufTypes::MessagePtr createEmptyConfigProto() override;
+  std::string name() const override;
+};
+
+} // namespace Formatter
+} // namespace Extensions
+} // namespace Envoy

source/extensions/formatter/req_without_query/req_without_query.cc

@@ -0,0 +1,88 @@
+#include "source/extensions/formatter/req_without_query/req_without_query.h"
+
+#include <string>
+
+#include "source/common/http/utility.h"
+#include "source/common/protobuf/utility.h"
+
+namespace Envoy {
+namespace Extensions {
+namespace Formatter {
+
+namespace {
+
+void truncate(std::string& str, absl::optional<uint32_t> max_length) {
+  if (!max_length) {
+    return;
+  }
+
+  str = str.substr(0, max_length.value());
+}
+
+} // namespace
+
+ReqWithoutQuery::ReqWithoutQuery(const std::string& main_header,
+                                 const std::string& alternative_header,
+                                 absl::optional<size_t> max_length)
+    : main_header_(main_header), alternative_header_(alternative_header), max_length_(max_length) {}
+
+absl::optional<std::string> ReqWithoutQuery::format(const Http::RequestHeaderMap& request,
+                                                    const Http::ResponseHeaderMap&,
+                                                    const Http::ResponseTrailerMap&,
+                                                    const StreamInfo::StreamInfo&,
+                                                    absl::string_view) const {
+  const Http::HeaderEntry* header = findHeader(request);
+  if (!header) {
+    return absl::nullopt;
+  }
+
+  std::string val = Http::Utility::stripQueryString(header->value());
+  truncate(val, max_length_);
+
+  return val;
+}
+
+ProtobufWkt::Value ReqWithoutQuery::formatValue(const Http::RequestHeaderMap& request,
+                                                const Http::ResponseHeaderMap&,
+                                                const Http::ResponseTrailerMap&,
+                                                const StreamInfo::StreamInfo&,
+                                                absl::string_view) const {
+  const Http::HeaderEntry* header = findHeader(request);
+  if (!header) {
+    return ValueUtil::nullValue();
+  }
+
+  std::string val = Http::Utility::stripQueryString(header->value());
+  truncate(val, max_length_);
+  return ValueUtil::stringValue(val);
+}
+
+const Http::HeaderEntry* ReqWithoutQuery::findHeader(const Http::HeaderMap& headers) const {
+  const auto header = headers.get(main_header_);
+
+  if (header.empty() && !alternative_header_.get().empty()) {
+    const auto alternate_header = headers.get(alternative_header_);
+    // TODO(https://github.com/envoyproxy/envoy/issues/13454): Potentially log all header values.
+    return alternate_header.empty() ? nullptr : alternate_header[0];
+  }
+
+  return header.empty() ? nullptr : header[0];
+}
+
+::Envoy::Formatter::FormatterProviderPtr
+ReqWithoutQueryCommandParser::parse(const std::string& token, size_t, size_t) const {
+  if (absl::StartsWith(token, "REQ_WITHOUT_QUERY(")) {
+    std::string main_header, alternative_header;
+    absl::optional<size_t> max_length;
+
+    Envoy::Formatter::SubstitutionFormatParser::parseCommandHeader(
+        token, ReqWithoutQueryParamStart, main_header, alternative_header, max_length);
+    return std::make_unique<ReqWithoutQuery>(main_header, alternative_header, max_length);
+  }
+
+  return nullptr;
+}
+
+} // namespace Formatter
+} // namespace Extensions
+} // namespace Envoy