ext_authz: support for buffering request body

api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto

@@ -50,6 +50,23 @@ message ExtAuthz {
   // semantically compatible. Deprecation note: This field is deprecated and should only be used for
   // version upgrade. See release notes for more details.
   bool use_alpha = 4 [deprecated = true];
+
+  // Enables filter to buffer the client request body and send it within the authorization request.
+  BufferSettings with_request_body = 5;
+}
+
+// Configuration for buffering the request data.
+message BufferSettings {
+  // Sets the maximum size of a message body that the filter will hold in memory. Envoy will return
+  // *HTTP 413* and will *not* initiate the authorization process when buffer reaches the number
+  // set in this field. Note that this setting will have precedence over :ref:`failure_mode_allow
+  // <envoy_api_field_config.filter.http.ext_authz.v2.ExtAuthz.failure_mode_allow>`.
+  uint32 max_request_bytes = 1 [(validate.rules).uint32.gt = 0];
+
+  // When this field is true, Envoy will buffer the message until *max_request_bytes* is reached.
+  // The authorization request will be dispatched and no 413 HTTP error will be returned by the
+  // filter.
+  bool allow_partial_message = 2;
 }
 
 // HttpService is used for raw HTTP communication between the filter and the authorization service.

api/envoy/service/auth/v2/BUILD

@@ -9,6 +9,7 @@ api_proto_library_internal(
     ],
     deps = [
         "//envoy/api/v2/core:address",
+        "//envoy/api/v2/core:base",
     ],
 )
 

api/envoy/service/auth/v2/attribute_context.proto

@@ -6,6 +6,7 @@ option java_outer_classname = "AttributeContextProto";
 option java_multiple_files = true;
 option java_package = "io.envoyproxy.envoy.service.auth.v2";
 
+import "envoy/api/v2/core/base.proto";
 import "envoy/api/v2/core/address.proto";
 
 import "google/protobuf/timestamp.proto";
@@ -112,6 +113,9 @@ message AttributeContext {
     // See :repo:`headers.h:ProtocolStrings <source/common/http/headers.h>` for a list of all
     // possible values.
     string protocol = 10;
+
+    // The HTTP request body.
+    string body = 11;
   }
 
   // The source of a network activity, such as starting a TCP connection.

docs/root/intro/version_history.rst

@@ -24,8 +24,10 @@ Version history
 * config: use Envoy cpuset size to set the default number or worker threads if :option:`--cpuset-threads` is enabled.
 * config: added support for :ref:`initial_fetch_timeout <envoy_api_field_core.ConfigSource.initial_fetch_timeout>`. The timeout is disabled by default.
 * cors: added :ref:`filter_enabled & shadow_enabled RuntimeFractionalPercent flags <cors-runtime>` to filter.
-* ext_authz: added an configurable option to make the gRPC service cross-compatible with V2Alpha. Note that this feature is already deprecated. It should be used for a short time, and only when transitioning from alpha to V2 release version.
-* ext_authz: migrated from V2alpha to V2 and improved the documentation.
+* ext_authz: added support for buffering request body.
+* ext_authz: migrated from v2alpha to v2 and improved docs.
+* ext_authz: added a configurable option to make the gRPC service cross-compatible with V2Alpha. Note that this feature is already deprecated. It should be used for a short time, and only when transitioning from alpha to V2 release version.
+* ext_authz: migrated from v2alpha to v2 and improved the documentation.
 * ext_authz: authorization request and response configuration has been separated into two distinct objects: :ref:`authorization request
   <envoy_api_field_config.filter.http.ext_authz.v2.HttpService.authorization_request>` and :ref:`authorization response
   <envoy_api_field_config.filter.http.ext_authz.v2.HttpService.authorization_response>`. In addition, :ref:`client headers

source/common/http/headers.h

@@ -172,8 +172,9 @@ class HeaderValues {
     const std::string Connect{"CONNECT"};
     const std::string Get{"GET"};
     const std::string Head{"HEAD"};
-    const std::string Post{"POST"};
     const std::string Options{"OPTIONS"};
+    const std::string Post{"POST"};
+    const std::string Trace{"TRACE"};
   } MethodValues;
 
   struct {

source/extensions/filters/common/ext_authz/check_request_utils.cc

@@ -76,7 +76,7 @@ std::string CheckRequestUtils::getHeaderStr(const Envoy::Http::HeaderEntry* entr
 void CheckRequestUtils::setHttpRequest(
     ::envoy::service::auth::v2::AttributeContext_HttpRequest& httpreq,
     const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
-    const Envoy::Http::HeaderMap& headers) {
+    const Envoy::Http::HeaderMap& headers, uint64_t max_request_bytes) {
 
   // Set id
   // The streamId is not qualified as a const. Although it is as it does not modify the object.
@@ -116,20 +116,29 @@ void CheckRequestUtils::setHttpRequest(
         return Envoy::Http::HeaderMap::Iterate::Continue;
       },
       mutable_headers);
+
+  // Set request body.
+  const Buffer::Instance* buffer = sdfc->decodingBuffer();
+  if (max_request_bytes > 0 && buffer != nullptr) {
+    const uint64_t length = std::min(buffer->length(), max_request_bytes);
+    std::string data(length, 0);
+    buffer->copyOut(0, length, &data[0]);
+    httpreq.set_body(std::move(data));
+  }
 }
 
 void CheckRequestUtils::setAttrContextRequest(
     ::envoy::service::auth::v2::AttributeContext_Request& req,
     const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
-    const Envoy::Http::HeaderMap& headers) {
-  setHttpRequest(*req.mutable_http(), callbacks, headers);
+    const Envoy::Http::HeaderMap& headers, uint64_t max_request_bytes) {
+  setHttpRequest(*req.mutable_http(), callbacks, headers, max_request_bytes);
 }
 
 void CheckRequestUtils::createHttpCheck(
     const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
     const Envoy::Http::HeaderMap& headers,
     Protobuf::Map<ProtobufTypes::String, ProtobufTypes::String>&& context_extensions,
-    envoy::service::auth::v2::CheckRequest& request) {
+    envoy::service::auth::v2::CheckRequest& request, uint64_t max_request_bytes) {
 
   auto attrs = request.mutable_attributes();
 
@@ -140,7 +149,7 @@ void CheckRequestUtils::createHttpCheck(
 
   setAttrContextPeer(*attrs->mutable_source(), *cb->connection(), service, false);
   setAttrContextPeer(*attrs->mutable_destination(), *cb->connection(), "", true);
-  setAttrContextRequest(*attrs->mutable_request(), callbacks, headers);
+  setAttrContextRequest(*attrs->mutable_request(), callbacks, headers, max_request_bytes);
 
   // Fill in the context extensions:
   (*attrs->mutable_context_extensions()) = std::move(context_extensions);

source/extensions/filters/common/ext_authz/check_request_utils.h

@@ -43,20 +43,19 @@ class CheckRequestUtils {
    * @param headers supplies the header map with http headers that will be used to create the
    *        check request.
    * @param request is the reference to the check request that will be filled up.
-   *
+   * @param with_request_body when true, will add the request body to the check request.
    */
   static void
   createHttpCheck(const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
                   const Envoy::Http::HeaderMap& headers,
                   Protobuf::Map<ProtobufTypes::String, ProtobufTypes::String>&& context_extensions,
-                  envoy::service::auth::v2::CheckRequest& request);
+                  envoy::service::auth::v2::CheckRequest& request, uint64_t max_request_bytes);
 
   /**
    * createTcpCheck is used to extract the attributes from the network layer and fill them up
    * in the CheckRequest proto message.
    * @param callbacks supplies the network layer context from which data can be extracted.
    * @param request is the reference to the check request that will be filled up.
-   *
    */
   static void createTcpCheck(const Network::ReadFilterCallbacks* callbacks,
                              envoy::service::auth::v2::CheckRequest& request);
@@ -67,10 +66,11 @@ class CheckRequestUtils {
                                  const bool local);
   static void setHttpRequest(::envoy::service::auth::v2::AttributeContext_HttpRequest& httpreq,
                              const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
-                             const Envoy::Http::HeaderMap& headers);
+                             const Envoy::Http::HeaderMap& headers, uint64_t max_request_bytes);
   static void setAttrContextRequest(::envoy::service::auth::v2::AttributeContext_Request& req,
                                     const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
-                                    const Envoy::Http::HeaderMap& headers);
+                                    const Envoy::Http::HeaderMap& headers,
+                                    uint64_t max_request_bytes);
   static std::string getHeaderStr(const Envoy::Http::HeaderEntry* entry);
   static Envoy::Http::HeaderMap::Iterate fillHttpHeaders(const Envoy::Http::HeaderEntry&, void*);
 };

source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc

@@ -161,7 +161,17 @@ void RawHttpClientImpl::check(RequestCallbacks& callbacks,
   ASSERT(callbacks_ == nullptr);
   callbacks_ = &callbacks;
 
-  Http::HeaderMapPtr headers = std::make_unique<Http::HeaderMapImpl>(lengthZeroHeader());
+  Http::HeaderMapPtr headers;
+  const uint64_t request_length = request.attributes().request().http().body().size();
+  if (request_length > 0) {
+    headers =
+        std::make_unique<Http::HeaderMapImpl,
+                         std::initializer_list<std::pair<Http::LowerCaseString, std::string>>>(
+            {{Http::Headers::get().ContentLength, std::to_string(request_length)}});
+  } else {
+    headers = std::make_unique<Http::HeaderMapImpl>(lengthZeroHeader());
+  }
+
   for (const auto& header : request.attributes().request().http().headers()) {
     const Http::LowerCaseString key{header.first};
     if (config_->requestHeaderMatchers()->matches(key.get())) {
@@ -179,8 +189,14 @@ void RawHttpClientImpl::check(RequestCallbacks& callbacks,
     headers->setReference(header_to_add.first, header_to_add.second);
   }
 
+  Http::MessagePtr message = std::make_unique<Envoy::Http::RequestMessageImpl>(std::move(headers));
+  if (request_length > 0) {
+    message->body() =
+        std::make_unique<Buffer::OwnedImpl>(request.attributes().request().http().body());
+  }
+
   request_ = cm_.httpAsyncClientForCluster(config_->cluster())
-                 .send(std::make_unique<Envoy::Http::RequestMessageImpl>(std::move(headers)), *this,
+                 .send(std::move(message), *this,
                        Http::AsyncClient::RequestOptions().setTimeout(config_->timeout()));
 }
 

source/extensions/filters/http/ext_authz/BUILD

@@ -24,6 +24,7 @@ envoy_cc_library(
         "//source/common/common:matchers_lib",
         "//source/common/common:minimal_logger_lib",
         "//source/common/http:codes_lib",
+        "//source/common/http:utility_lib",
         "//source/common/router:config_lib",
         "//source/extensions/filters/common/ext_authz:ext_authz_grpc_lib",
         "//source/extensions/filters/common/ext_authz:ext_authz_http_lib",

source/extensions/filters/http/ext_authz/ext_authz.cc

@@ -3,6 +3,7 @@
 #include "common/common/assert.h"
 #include "common/common/enum_to_int.h"
 #include "common/http/codes.h"
+#include "common/http/utility.h"
 #include "common/router/config_impl.h"
 
 #include "extensions/filters/http/well_known_names.h"
@@ -54,31 +55,57 @@ void Filter::initiateCall(const Http::HeaderMap& headers) {
     context_extensions = maybe_merged_per_route_config.value().takeContextExtensions();
   }
   Filters::Common::ExtAuthz::CheckRequestUtils::createHttpCheck(
-      callbacks_, headers, std::move(context_extensions), check_request_);
+      callbacks_, headers, std::move(context_extensions), check_request_,
+      config_->maxRequestBytes());
 
+  ENVOY_STREAM_LOG(trace, "ext_authz filter calling authorization server", *callbacks_);
   state_ = State::Calling;
-  // Don't let the filter chain continue as we are going to invoke check call.
-  filter_return_ = FilterReturn::StopDecoding;
+  filter_return_ = FilterReturn::StopDecoding; // Don't let the filter chain continue as we are
+                                               // going to invoke check call.
   initiating_call_ = true;
-  ENVOY_STREAM_LOG(trace, "ext_authz filter calling authorization server", *callbacks_);
   client_->check(*this, check_request_, callbacks_->activeSpan());
   initiating_call_ = false;
 }
 
-Http::FilterHeadersStatus Filter::decodeHeaders(Http::HeaderMap& headers, bool) {
+Http::FilterHeadersStatus Filter::decodeHeaders(Http::HeaderMap& headers, bool end_stream) {
   request_headers_ = &headers;
+  buffer_data_ = config_->withRequestBody() &&
+                 !(end_stream || Http::Utility::isWebSocketUpgradeRequest(headers) ||
+                   Http::Utility::isH2UpgradeRequest(headers));
+  if (buffer_data_) {
+    ENVOY_STREAM_LOG(debug, "ext_authz filter is buffering the request", *callbacks_);
+    if (!config_->allowPartialMessage()) {
+      callbacks_->setDecoderBufferLimit(config_->maxRequestBytes());
+    }
+    return Http::FilterHeadersStatus::StopIteration;
+  }
+
   initiateCall(headers);
   return filter_return_ == FilterReturn::StopDecoding ? Http::FilterHeadersStatus::StopIteration
                                                       : Http::FilterHeadersStatus::Continue;
 }
 
-Http::FilterDataStatus Filter::decodeData(Buffer::Instance&, bool) {
+Http::FilterDataStatus Filter::decodeData(Buffer::Instance&, bool end_stream) {
+  if (buffer_data_) {
+    if (end_stream || isBufferFull()) {
+      ENVOY_STREAM_LOG(debug, "ext_authz filter finished buffering the request", *callbacks_);
+      initiateCall(*request_headers_);
+    } else {
+      return Http::FilterDataStatus::StopIterationAndBuffer;
+    }
+  }
+
   return filter_return_ == FilterReturn::StopDecoding
              ? Http::FilterDataStatus::StopIterationAndWatermark
              : Http::FilterDataStatus::Continue;
 }
 
 Http::FilterTrailersStatus Filter::decodeTrailers(Http::HeaderMap&) {
+  if (buffer_data_ && filter_return_ != FilterReturn::StopDecoding) {
+    ENVOY_STREAM_LOG(debug, "ext_authz filter finished buffering the request", *callbacks_);
+    initiateCall(*request_headers_);
+  }
+
   return filter_return_ == FilterReturn::StopDecoding ? Http::FilterTrailersStatus::StopIteration
                                                       : Http::FilterTrailersStatus::Continue;
 }
@@ -122,29 +149,30 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) {
     break;
   }
 
-  ENVOY_STREAM_LOG(trace, "ext_authz received status code {}", *callbacks_,
+  ENVOY_STREAM_LOG(trace, "ext_authz filter received status code {}", *callbacks_,
                    enumToInt(response->status_code));
 
   // We fail open/fail close based of filter config
   // if there is an error contacting the service.
   if (response->status == CheckStatus::Denied ||
       (response->status == CheckStatus::Error && !config_->failureModeAllow())) {
-    ENVOY_STREAM_LOG(debug, "ext_authz rejected the request", *callbacks_);
-    ENVOY_STREAM_LOG(trace, "ext_authz downstream header(s):", *callbacks_);
-    callbacks_->sendLocalReply(response->status_code, response->body,
-                               [& headers = response->headers_to_add, &callbacks = *callbacks_](
-                                   Http::HeaderMap& response_headers) -> void {
-                                 for (const auto& header : headers) {
-                                   response_headers.remove(header.first);
-                                   response_headers.addCopy(header.first, header.second);
-                                   ENVOY_STREAM_LOG(trace, " '{}':'{}'", callbacks,
-                                                    header.first.get(), header.second);
-                                 }
-                               },
-                               absl::nullopt);
+    ENVOY_STREAM_LOG(debug, "ext_authz filter rejected the request", *callbacks_);
+    callbacks_->sendLocalReply(
+        response->status_code, response->body,
+        [& headers = response->headers_to_add,
+         &callbacks = *callbacks_](Http::HeaderMap& response_headers) -> void {
+          ENVOY_STREAM_LOG(trace,
+                           "ext_authz filter added header(s) to the local response:", callbacks);
+          for (const auto& header : headers) {
+            response_headers.remove(header.first);
+            response_headers.addCopy(header.first, header.second);
+            ENVOY_STREAM_LOG(trace, " '{}':'{}'", callbacks, header.first.get(), header.second);
+          }
+        },
+        absl::nullopt);
     callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::UnauthorizedExternalService);
   } else {
-    ENVOY_STREAM_LOG(debug, "ext_authz accepted the request", *callbacks_);
+    ENVOY_STREAM_LOG(debug, "ext_authz filter accepted the request", *callbacks_);
     // Let the filter chain continue.
     filter_return_ = FilterReturn::ContinueDecoding;
     if (config_->failureModeAllow() && response->status == CheckStatus::Error) {
@@ -153,7 +181,7 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) {
     }
     // Only send headers if the response is ok.
     if (response->status == CheckStatus::OK) {
-      ENVOY_STREAM_LOG(trace, "ext_authz upstream header(s):", *callbacks_);
+      ENVOY_STREAM_LOG(trace, "ext_authz filter added header(s) to the request:", *callbacks_);
       for (const auto& header : response->headers_to_add) {
         Http::HeaderEntry* header_to_modify = request_headers_->get(header.first);
         if (header_to_modify) {
@@ -179,6 +207,14 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) {
   }
 }
 
+bool Filter::isBufferFull() {
+  const auto* buffer = callbacks_->decodingBuffer();
+  if (config_->allowPartialMessage() && buffer != nullptr) {
+    return buffer->length() >= config_->maxRequestBytes();
+  }
+  return false;
+}
+
 } // namespace ExtAuthz
 } // namespace HttpFilters
 } // namespace Extensions