ext_authz: support for buffering request body

api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto

@@ -50,6 +50,23 @@ message ExtAuthz {
   // semantically compatible. Deprecation note: This field is deprecated and should only be used for
   // version upgrade. See release notes for more details.
   bool use_alpha = 4 [deprecated = true];
+
+  // Enables filter to buffer the client request body and send it within the authorization request.
+  BufferSettings with_request_body = 5;
+}
+
+// Configuration for buffering the request data.
+message BufferSettings {
+  // Sets the maximum size of a message body that the filter will hold in memory. Envoy will return
+  // *HTTP 413* and will *not* initiate the authorization process when buffer reaches the number
+  // set in this field. Note that this setting will have precedence over :ref:`failure_mode_allow
+  // <envoy_api_field_config.filter.http.ext_authz.v2.ExtAuthz.failure_mode_allow>`.
+  uint32 max_request_bytes = 1 [(validate.rules).uint32.gt = 0];
+
+  // When this field is true, Envoy will buffer the message until *max_request_bytes* is reached.
+  // The authorization request will be dispatched and no 413 HTTP error will be returned by the
+  // filter.
+  bool allow_partial_message = 2;
 }
 
 // HttpService is used for raw HTTP communication between the filter and the authorization service.

api/envoy/service/auth/v2/BUILD

@@ -9,6 +9,7 @@ api_proto_library_internal(
     ],
     deps = [
         "//envoy/api/v2/core:address",
+        "//envoy/api/v2/core:base",
     ],
 )
 

api/envoy/service/auth/v2/attribute_context.proto

@@ -6,6 +6,7 @@ option java_outer_classname = "AttributeContextProto";
 option java_multiple_files = true;
 option java_package = "io.envoyproxy.envoy.service.auth.v2";
 
+import "envoy/api/v2/core/base.proto";
 import "envoy/api/v2/core/address.proto";
 
 import "google/protobuf/timestamp.proto";
@@ -108,6 +109,9 @@ message AttributeContext {
     // The network protocol used with the request, such as
     // "http/1.1", "spdy/3", "h2", "h2c"
     string protocol = 10;
+
+    // The HTTP request body.
+    string body = 11;
   }
 
   // The source of a network activity, such as starting a TCP connection.

docs/root/intro/version_history.rst

@@ -20,6 +20,9 @@ Version history
 * config: use Envoy cpuset size to set the default number or worker threads if :option:`--cpuset-threads` is enabled.
 * config: added support for :ref:`initial_fetch_timeout <envoy_api_field_core.ConfigSource.initial_fetch_timeout>`. The timeout is disabled by default.
 * cors: added :ref:`filter_enabled & shadow_enabled RuntimeFractionalPercent flags <cors-runtime>` to filter.
+* ext_authz: added support for buffering request body.
+* ext_authz: migrated from V2alpha to V2 and improved docs.
+* ext_authz: added an configurable option to make the gRPC service cross-compatible with V2Alpha. Note that this feature is already deprecated. It should be used for a short time, and only when transitioning from alpha to V2 release version. 
 * ext_authz: added an configurable option to make the gRPC service cross-compatible with V2Alpha. Note that this feature is already deprecated. It should be used for a short time, and only when transitioning from alpha to V2 release version.
 * ext_authz: migrated from V2alpha to V2 and improved the documentation.
 * ext_authz: authorization request and response configuration has been separated into two distinct objects: :ref:`authorization request

source/common/http/headers.h

@@ -172,8 +172,9 @@ class HeaderValues {
     const std::string Connect{"CONNECT"};
     const std::string Get{"GET"};
     const std::string Head{"HEAD"};
-    const std::string Post{"POST"};
     const std::string Options{"OPTIONS"};
+    const std::string Post{"POST"};
+    const std::string Trace{"TRACE"};
   } MethodValues;
 
   struct {

source/extensions/filters/common/ext_authz/check_request_utils.cc

@@ -74,7 +74,7 @@ std::string CheckRequestUtils::getHeaderStr(const Envoy::Http::HeaderEntry* entr
 void CheckRequestUtils::setHttpRequest(
     ::envoy::service::auth::v2::AttributeContext_HttpRequest& httpreq,
     const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
-    const Envoy::Http::HeaderMap& headers) {
+    const Envoy::Http::HeaderMap& headers, uint64_t max_request_bytes) {
 
   // Set id
   // The streamId is not qualified as a const. Although it is as it does not modify the object.
@@ -114,20 +114,29 @@ void CheckRequestUtils::setHttpRequest(
         return Envoy::Http::HeaderMap::Iterate::Continue;
       },
       mutable_headers);
+
+  // Set request body.
+  const Buffer::Instance* buffer = sdfc->decodingBuffer();
+  if (max_request_bytes > 0 && buffer != nullptr) {
+    const uint64_t length = std::min(buffer->length(), max_request_bytes);
+    std::string data(length, 0);
+    buffer->copyOut(0, length, &data[0]);
+    httpreq.set_body(std::move(data));
+  }
 }
 
 void CheckRequestUtils::setAttrContextRequest(
     ::envoy::service::auth::v2::AttributeContext_Request& req,
     const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
-    const Envoy::Http::HeaderMap& headers) {
-  setHttpRequest(*req.mutable_http(), callbacks, headers);
+    const Envoy::Http::HeaderMap& headers, uint64_t max_request_bytes) {
+  setHttpRequest(*req.mutable_http(), callbacks, headers, max_request_bytes);
 }
 
 void CheckRequestUtils::createHttpCheck(
     const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
     const Envoy::Http::HeaderMap& headers,
     Protobuf::Map<ProtobufTypes::String, ProtobufTypes::String>&& context_extensions,
-    envoy::service::auth::v2::CheckRequest& request) {
+    envoy::service::auth::v2::CheckRequest& request, uint64_t max_request_bytes) {
 
   auto attrs = request.mutable_attributes();
 
@@ -138,7 +147,7 @@ void CheckRequestUtils::createHttpCheck(
 
   setAttrContextPeer(*attrs->mutable_source(), *cb->connection(), service, false);
   setAttrContextPeer(*attrs->mutable_destination(), *cb->connection(), "", true);
-  setAttrContextRequest(*attrs->mutable_request(), callbacks, headers);
+  setAttrContextRequest(*attrs->mutable_request(), callbacks, headers, max_request_bytes);
 
   // Fill in the context extensions:
   (*attrs->mutable_context_extensions()) = std::move(context_extensions);

source/extensions/filters/common/ext_authz/check_request_utils.h

@@ -43,20 +43,19 @@ class CheckRequestUtils {
    * @param headers supplies the header map with http headers that will be used to create the
    *        check request.
    * @param request is the reference to the check request that will be filled up.
-   *
+   * @param with_request_body when true, will add the request body to the check request.
    */
   static void
   createHttpCheck(const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
                   const Envoy::Http::HeaderMap& headers,
                   Protobuf::Map<ProtobufTypes::String, ProtobufTypes::String>&& context_extensions,
-                  envoy::service::auth::v2::CheckRequest& request);
+                  envoy::service::auth::v2::CheckRequest& request, uint64_t max_request_bytes);
 
   /**
    * createTcpCheck is used to extract the attributes from the network layer and fill them up
    * in the CheckRequest proto message.
    * @param callbacks supplies the network layer context from which data can be extracted.
    * @param request is the reference to the check request that will be filled up.
-   *
    */
   static void createTcpCheck(const Network::ReadFilterCallbacks* callbacks,
                              envoy::service::auth::v2::CheckRequest& request);
@@ -67,10 +66,11 @@ class CheckRequestUtils {
                                  const bool local);
   static void setHttpRequest(::envoy::service::auth::v2::AttributeContext_HttpRequest& httpreq,
                              const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
-                             const Envoy::Http::HeaderMap& headers);
+                             const Envoy::Http::HeaderMap& headers, uint64_t max_request_bytes);
   static void setAttrContextRequest(::envoy::service::auth::v2::AttributeContext_Request& req,
                                     const Envoy::Http::StreamDecoderFilterCallbacks* callbacks,
-                                    const Envoy::Http::HeaderMap& headers);
+                                    const Envoy::Http::HeaderMap& headers,
+                                    uint64_t max_request_bytes);
   static std::string getHeaderStr(const Envoy::Http::HeaderEntry* entry);
   static Envoy::Http::HeaderMap::Iterate fillHttpHeaders(const Envoy::Http::HeaderEntry&, void*);
 };

source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc

@@ -161,7 +161,17 @@ void RawHttpClientImpl::check(RequestCallbacks& callbacks,
   ASSERT(callbacks_ == nullptr);
   callbacks_ = &callbacks;
 
-  Http::HeaderMapPtr headers = std::make_unique<Http::HeaderMapImpl>(lengthZeroHeader());
+  Http::HeaderMapPtr headers;
+  const uint64_t request_length = request.attributes().request().http().body().size();
+  if (request_length > 0) {
+    headers =
+        std::make_unique<Http::HeaderMapImpl,
+                         std::initializer_list<std::pair<Http::LowerCaseString, std::string>>>(
+            {{Http::Headers::get().ContentLength, std::to_string(request_length)}});
+  } else {
+    headers = std::make_unique<Http::HeaderMapImpl>(lengthZeroHeader());
+  }
+
   for (const auto& header : request.attributes().request().http().headers()) {
     const Http::LowerCaseString key{header.first};
     if (config_->requestHeaderMatchers()->matches(key.get())) {
@@ -179,8 +189,14 @@ void RawHttpClientImpl::check(RequestCallbacks& callbacks,
     headers->setReference(header_to_add.first, header_to_add.second);
   }
 
+  Http::MessagePtr message = std::make_unique<Envoy::Http::RequestMessageImpl>(std::move(headers));
+  if (request_length > 0) {
+    message->body() =
+        std::make_unique<Buffer::OwnedImpl>(request.attributes().request().http().body());
+  }
+
   request_ = cm_.httpAsyncClientForCluster(config_->cluster())
-                 .send(std::make_unique<Envoy::Http::RequestMessageImpl>(std::move(headers)), *this,
+                 .send(std::move(message), *this,
                        Http::AsyncClient::RequestOptions().setTimeout(config_->timeout()));
 }
 

source/extensions/filters/http/ext_authz/ext_authz.cc

@@ -7,11 +7,26 @@
 
 #include "extensions/filters/http/well_known_names.h"
 
+#include "absl/container/flat_hash_set.h"
+
 namespace Envoy {
 namespace Extensions {
 namespace HttpFilters {
 namespace ExtAuthz {
 
+namespace {
+// Asserts that the HTTP method is for a header-only request. For details
+// https://tools.ietf.org/html/rfc7231#section-4.3.
+bool isHeaderOnlyMethod(absl::string_view method) {
+  const static absl::flat_hash_set<std::string>* keys = new absl::flat_hash_set<std::string>(
+      {Http::Headers::get().MethodValues.Get, Http::Headers::get().MethodValues.Head,
+       Http::Headers::get().MethodValues.Connect, Http::Headers::get().MethodValues.Options,
+       Http::Headers::get().MethodValues.Trace});
+
+  return keys->find(method) != keys->end();
+}
+} // namespace
+
 void FilterConfigPerRoute::merge(const FilterConfigPerRoute& other) {
   disabled_ = other.disabled_;
   auto begin_it = other.context_extensions_.begin();
@@ -54,7 +69,8 @@ void Filter::initiateCall(const Http::HeaderMap& headers) {
     context_extensions = maybe_merged_per_route_config.value().takeContextExtensions();
   }
   Filters::Common::ExtAuthz::CheckRequestUtils::createHttpCheck(
-      callbacks_, headers, std::move(context_extensions), check_request_);
+      callbacks_, headers, std::move(context_extensions), check_request_,
+      config_->maxRequestBytes());
 
   state_ = State::Calling;
   // Don't let the filter chain continue as we are going to invoke check call.
@@ -65,20 +81,45 @@ void Filter::initiateCall(const Http::HeaderMap& headers) {
   initiating_call_ = false;
 }
 
-Http::FilterHeadersStatus Filter::decodeHeaders(Http::HeaderMap& headers, bool) {
+Http::FilterHeadersStatus Filter::decodeHeaders(Http::HeaderMap& headers, bool end_stream) {
   request_headers_ = &headers;
+  const auto* method = headers.Method();
+  buffer_data_ = config_->withRequestBody() &&
+                 !(end_stream || (method && isHeaderOnlyMethod(method->value().getStringView())));
+
+  if (buffer_data_) {
+    ENVOY_STREAM_LOG(debug, "ext_authz is buffering", *callbacks_);
+    if (!config_->allowPartialMessage()) {
+      callbacks_->setDecoderBufferLimit(config_->maxRequestBytes());
+    }
+    return Http::FilterHeadersStatus::StopIteration;
+  }
+
   initiateCall(headers);
   return filter_return_ == FilterReturn::StopDecoding ? Http::FilterHeadersStatus::StopIteration
                                                       : Http::FilterHeadersStatus::Continue;
 }
 
-Http::FilterDataStatus Filter::decodeData(Buffer::Instance&, bool) {
+Http::FilterDataStatus Filter::decodeData(Buffer::Instance&, bool end_stream) {
+  if (buffer_data_) {
+    if (!end_stream && !isBufferFull()) {
+      return Http::FilterDataStatus::StopIterationAndBuffer;
+    }
+    return Http::FilterDataStatus::StopIterationAndWatermark;
+  }
+
   return filter_return_ == FilterReturn::StopDecoding
              ? Http::FilterDataStatus::StopIterationAndWatermark
              : Http::FilterDataStatus::Continue;
 }
 
 Http::FilterTrailersStatus Filter::decodeTrailers(Http::HeaderMap&) {
+  if (buffer_data_) {
+    // Call the authorization service.
+    ENVOY_STREAM_LOG(debug, "ext_authz finished buffering", *callbacks_);
+    initiateCall(*request_headers_);
+  }
+
   return filter_return_ == FilterReturn::StopDecoding ? Http::FilterTrailersStatus::StopIteration
                                                       : Http::FilterTrailersStatus::Continue;
 }
@@ -179,6 +220,13 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) {
   }
 }
 
+bool Filter::isBufferFull() {
+  if (config_->allowPartialMessage()) {
+    return callbacks_->decodingBuffer()->length() >= config_->maxRequestBytes();
+  }
+  return false;
+}
+
 } // namespace ExtAuthz
 } // namespace HttpFilters
 } // namespace Extensions

source/extensions/filters/http/ext_authz/ext_authz.h

@@ -39,18 +39,31 @@ class FilterConfig {
   FilterConfig(const envoy::config::filter::http::ext_authz::v2::ExtAuthz& config,
                const LocalInfo::LocalInfo& local_info, Stats::Scope& scope,
                Runtime::Loader& runtime, Http::Context& http_context)
-      : failure_mode_allow_(config.failure_mode_allow()), local_info_(local_info), scope_(scope),
-        runtime_(runtime), http_context_(http_context) {}
+      : allow_partial_message_(config.with_request_body().allow_partial_message()),
+        failure_mode_allow_(config.failure_mode_allow()),
+        max_request_bytes_(config.with_request_body().max_request_bytes()), local_info_(local_info),
+        scope_(scope), runtime_(runtime), http_context_(http_context) {}
+
+  bool allowPartialMessage() const { return allow_partial_message_; }
+
+  bool withRequestBody() const { return max_request_bytes_ > 0; }
 
   bool failureModeAllow() const { return failure_mode_allow_; }
+
+  uint32_t maxRequestBytes() const { return max_request_bytes_; }
+
   const LocalInfo::LocalInfo& localInfo() const { return local_info_; }
+
   Runtime::Loader& runtime() { return runtime_; }
+
   Stats::Scope& scope() { return scope_; }
 
   Http::Context& httpContext() { return http_context_; }
 
 private:
-  bool failure_mode_allow_{};
+  const bool allow_partial_message_;
+  const bool failure_mode_allow_;
+  const uint32_t max_request_bytes_;
   const LocalInfo::LocalInfo& local_info_;
   Stats::Scope& scope_;
   Runtime::Loader& runtime_;
@@ -116,6 +129,8 @@ class Filter : public Logger::Loggable<Logger::Id::filter>,
 
 private:
   void addResponseHeaders(Http::HeaderMap& header_map, const Http::HeaderVector& headers);
+  void initiateCall(const Http::HeaderMap& headers);
+  bool isBufferFull();
 
   // State of this filter's communication with the external authorization service.
   // The filter has either not started calling the external service, in the middle of calling
@@ -127,7 +142,6 @@ class Filter : public Logger::Loggable<Logger::Id::filter>,
   // the filter chain should stop. Otherwise the filter chain can continue to the next filter.
   enum class FilterReturn { ContinueDecoding, StopDecoding };
 
-  void initiateCall(const Http::HeaderMap& headers);
   Http::HeaderMapPtr getHeaderMap(const Filters::Common::ExtAuthz::ResponsePtr& response);
   FilterConfigSharedPtr config_;
   Filters::Common::ExtAuthz::ClientPtr client_;
@@ -139,6 +153,7 @@ class Filter : public Logger::Loggable<Logger::Id::filter>,
 
   // Used to identify if the callback to onComplete() is synchronous (on the stack) or asynchronous.
   bool initiating_call_{};
+  bool buffer_data_{};
   envoy::service::auth::v2::CheckRequest check_request_{};
 };