http1 encode trailers in chunk encoding

api/envoy/api/v2/core/protocol.proto

@@ -46,6 +46,7 @@ message HttpProtocolOptions {
   google.protobuf.UInt32Value max_headers_count = 2 [(validate.rules).uint32 = {gte: 1}];
 }
 
+// [#next-free-field: 6]
 message Http1ProtocolOptions {
   message HeaderKeyFormat {
     message ProperCaseWords {
@@ -83,6 +84,9 @@ message Http1ProtocolOptions {
   // Describes how the keys for response headers should be formatted. By default, all header keys
   // are lower cased.
   HeaderKeyFormat header_key_format = 4;
+
+  // Enables trailers for HTTP1. By default the HTTP/1 codec drops proxied trailers.
+  bool enable_trailers = 5;
 }
 
 // [#next-free-field: 13]

api/envoy/api/v3alpha/core/protocol.proto

@@ -46,6 +46,7 @@ message HttpProtocolOptions {
   google.protobuf.UInt32Value max_headers_count = 2 [(validate.rules).uint32 = {gte: 1}];
 }
 
+// [#next-free-field: 6]
 message Http1ProtocolOptions {
   message HeaderKeyFormat {
     message ProperCaseWords {
@@ -83,6 +84,9 @@ message Http1ProtocolOptions {
   // Describes how the keys for response headers should be formatted. By default, all header keys
   // are lower cased.
   HeaderKeyFormat header_key_format = 4;
+
+  // Enables trailers for HTTP1. By default the HTTP/1 codec drops proxied trailers.
+  bool enable_trailers = 5;
 }
 
 // [#next-free-field: 13]

docs/root/intro/version_history.rst

@@ -9,20 +9,21 @@ Version history
 * build: official released binary is now built against libc++.
 * ext_authz: added :ref:`configurable ability<envoy_api_field_config.filter.http.ext_authz.v2.ExtAuthz.include_peer_certificate>` to send the :ref:`certificate<envoy_api_field_service.auth.v2.AttributeContext.Peer.certificate>` to the `ext_authz` service.
 * health check: gRPC health checker sets the gRPC deadline to the configured timeout duration.
+* http: add support for http1 trailers, to enable use :ref:`enable_trailers <envoy_api_field_core.Http1ProtocolOptions.enable_trailers>`.
 * http: added the ability to sanitize headers nominated by the Connection header. This new behavior is guarded by envoy.reloadable_features.connection_header_sanitization which defaults to true.
 * http: support :ref:`auto_host_rewrite_header<envoy_api_field_config.filter.http.dynamic_forward_proxy.v2alpha.PerRouteConfig.auto_host_rewrite_header>` in the dynamic forward proxy.
 * jwt_authn: added :ref:`bypass_cors_preflight<envoy_api_field_config.filter.http.jwt_authn.v2alpha.JwtAuthentication.bypass_cors_preflight>` to allow bypassing the CORS preflight request.
 * lb_subset_config: new fallback policy for selectors: :ref:`KEYS_SUBSET<envoy_api_enum_value_Cluster.LbSubsetConfig.LbSubsetSelector.LbSubsetSelectorFallbackPolicy.KEYS_SUBSET>`
 * logger: added :ref:`--log-format-escaped <operations_cli>` command line option to escape newline characters in application logs.
 * redis: performance improvement for larger split commands by avoiding string copies.
 * router: added support for REQ(header-name) :ref:`header formatter <config_http_conn_man_headers_custom_request_headers>`.
+* router: exposed DOWNSTREAM_REMOTE_ADDRESS as custom HTTP request/response headers.
 * router: skip the Location header when the response code is not a 201 or a 3xx.
 * server: fixed a bug in config validation for configs with runtime layers
 * tcp_proxy: added :ref:`ClusterWeight.metadata_match<envoy_api_field_config.filter.network.tcp_proxy.v2.TcpProxy.WeightedCluster.ClusterWeight.metadata_match>`
 * tcp_proxy: added :ref:`hash_policy<envoy_api_field_config.filter.network.tcp_proxy.v2.TcpProxy.hash_policy>`
 * thrift_proxy: added support for cluster header based routing.
 * tls: remove TLS 1.0 and 1.1 from client defaults
-* router: exposed DOWNSTREAM_REMOTE_ADDRESS as custom HTTP request/response headers.
 
 1.12.0 (October 31, 2019)
 =========================

include/envoy/http/codec.h

@@ -229,6 +229,12 @@ struct Http1Settings {
   bool accept_http_10_{false};
   // Set a default host if no Host: header is present for HTTP/1.0 requests.`
   std::string default_host_for_http_10_;
+  // Encode trailers in Http. By default the HTTP/1 codec drops proxied trailers.
+  // Note that this only happens when Envoy is chunk encoding which occurs when:
+  // The request is Http1.1
+  // Is neither a HEAD only request or a HTTP Upgrade
+  // Not a HEAD request
+  bool enable_trailers_{false};
 
   enum class HeaderKeyFormat {
     // By default no formatting is performed, presenting all headers in lowercase (as Envoy

source/common/http/http1/codec_impl.cc

@@ -40,7 +40,8 @@ HeaderKeyFormatterPtr formatter(const Http::Http1Settings& settings) {
 } // namespace
 
 const std::string StreamEncoderImpl::CRLF = "\r\n";
-const std::string StreamEncoderImpl::LAST_CHUNK = "0\r\n\r\n";
+// Last chunk as defined here https://tools.ietf.org/html/rfc7230#section-4.1
+const std::string StreamEncoderImpl::LAST_CHUNK = "0\r\n";
 
 StreamEncoderImpl::StreamEncoderImpl(ConnectionImpl& connection,
                                      HeaderKeyFormatter* header_key_formatter)
@@ -187,7 +188,31 @@ void StreamEncoderImpl::encodeData(Buffer::Instance& data, bool end_stream) {
   }
 }
 
-void StreamEncoderImpl::encodeTrailers(const HeaderMap&) { endEncode(); }
+void StreamEncoderImpl::encodeTrailers(const HeaderMap& trailers) {
+  if (!connection_.enable_trailers_) {
+    return endEncode();
+  }
+  // Trailers only matter if it is a chunk transfer encoding
+  // https://tools.ietf.org/html/rfc7230#section-4.4
+  if (chunk_encoding_) {
+    // Finalize the body
+    connection_.buffer().add(LAST_CHUNK);
+
+    trailers.iterate(
+        [](const HeaderEntry& header, void* context) -> HeaderMap::Iterate {
+          static_cast<StreamEncoderImpl*>(context)->encodeFormattedHeader(
+              header.key().getStringView(), header.value().getStringView());
+          return HeaderMap::Iterate::Continue;
+        },
+        this);
+
+    connection_.flushOutput();
+    connection_.buffer().add(CRLF);
+  }
+
+  connection_.flushOutput();
+  connection_.onEncodeComplete();
+}
 
 void StreamEncoderImpl::encodeMetadata(const MetadataMapVector&) {
   connection_.stats().metadata_not_supported_error_.inc();
@@ -196,6 +221,7 @@ void StreamEncoderImpl::encodeMetadata(const MetadataMapVector&) {
 void StreamEncoderImpl::endEncode() {
   if (chunk_encoding_) {
     connection_.buffer().add(LAST_CHUNK);
+    connection_.buffer().add(CRLF);
   }
 
   connection_.flushOutput();
@@ -353,8 +379,9 @@ const ToLowerTable& ConnectionImpl::toLowerTable() {
 ConnectionImpl::ConnectionImpl(Network::Connection& connection, Stats::Scope& stats,
                                http_parser_type type, uint32_t max_headers_kb,
                                const uint32_t max_headers_count,
-                               HeaderKeyFormatterPtr&& header_key_formatter)
-    : connection_(connection), stats_{ALL_HTTP1_CODEC_STATS(POOL_COUNTER_PREFIX(stats, "http1."))},
+                               HeaderKeyFormatterPtr&& header_key_formatter, bool enable_trailers)
+    : enable_trailers_(enable_trailers),
+      connection_(connection), stats_{ALL_HTTP1_CODEC_STATS(POOL_COUNTER_PREFIX(stats, "http1."))},
       header_key_formatter_(std::move(header_key_formatter)),
       output_buffer_([&]() -> void { this->onBelowLowWatermark(); },
                      [&]() -> void { this->onAboveHighWatermark(); }),
@@ -371,11 +398,13 @@ ConnectionImpl::ConnectionImpl(Network::Connection& connection, Stats::Scope& st
 void ConnectionImpl::completeLastHeader() {
   ENVOY_CONN_LOG(trace, "completed header: key={} value={}", connection_,
                  current_header_field_.getStringView(), current_header_value_.getStringView());
+
   if (!current_header_field_.empty()) {
     toLowerTable().toLowerCase(current_header_field_.buffer(), current_header_field_.size());
     current_header_map_->addViaMove(std::move(current_header_field_),
                                     std::move(current_header_value_));
   }
+
   // Check if the number of headers exceeds the limit.
   if (current_header_map_->size() > max_headers_count_) {
     error_code_ = Http::Code::RequestHeaderFieldsTooLarge;
@@ -449,11 +478,17 @@ size_t ConnectionImpl::dispatchSlice(const char* slice, size_t len) {
 }
 
 void ConnectionImpl::onHeaderField(const char* data, size_t length) {
+  ENVOY_CONN_LOG(trace, "onHeaderField: data={}", connection_, absl::string_view(data, length));
+  // We previously already finished up the headers, these headers are
+  // now trailers
   if (header_parsing_state_ == HeaderParsingState::Done) {
-    // Ignore trailers.
-    return;
+    if (!enable_trailers_) {
+      // Ignore trailers.
+      return;
+    }
+    processing_trailers_ = true;
+    header_parsing_state_ = HeaderParsingState::Field;
   }
-
   if (header_parsing_state_ == HeaderParsingState::Value) {
     completeLastHeader();
   }
@@ -462,12 +497,17 @@ void ConnectionImpl::onHeaderField(const char* data, size_t length) {
 }
 
 void ConnectionImpl::onHeaderValue(const char* data, size_t length) {
-  if (header_parsing_state_ == HeaderParsingState::Done) {
+  const absl::string_view header_value = absl::string_view(data, length);
+  ENVOY_CONN_LOG(trace, "onHeaderValue: data={}", connection_, header_value);
+
+  if (header_parsing_state_ == HeaderParsingState::Done && !enable_trailers_) {
     // Ignore trailers.
     return;
   }
 
-  const absl::string_view header_value = absl::string_view(data, length);
+  if (!current_header_map_) {
+    current_header_map_ = std::make_unique<HeaderMapImpl>();
+  }
 
   if (strict_header_validation_) {
     if (!Http::HeaderUtility::headerIsValid(header_value)) {
@@ -500,8 +540,10 @@ void ConnectionImpl::onHeaderValue(const char* data, size_t length) {
 }
 
 int ConnectionImpl::onHeadersCompleteBase() {
-  ENVOY_CONN_LOG(trace, "headers complete", connection_);
+  const std::string header_type = processing_trailers_ ? "trailers" : "headers";
+  ENVOY_CONN_LOG(trace, "onHeadersCompleteBase complete for {}", connection_, header_type);
   completeLastHeader();
+
   // Validate that the completed HeaderMap's cached byte size exists and is correct.
   // This assert iterates over the HeaderMap.
   ASSERT(current_header_map_->byteSize().has_value() &&
@@ -547,6 +589,7 @@ int ConnectionImpl::onHeadersCompleteBase() {
 
   int rc = onHeadersComplete(std::move(current_header_map_));
   current_header_map_.reset();
+
   header_parsing_state_ = HeaderParsingState::Done;
 
   // Returning 2 informs http_parser to not expect a body or further data on this connection.
@@ -555,6 +598,7 @@ int ConnectionImpl::onHeadersCompleteBase() {
 
 void ConnectionImpl::onMessageCompleteBase() {
   ENVOY_CONN_LOG(trace, "message complete", connection_);
+
   if (handling_upgrade_) {
     // If this is an upgrade request, swallow the onMessageComplete. The
     // upgrade payload will be treated as stream body.
@@ -563,7 +607,14 @@ void ConnectionImpl::onMessageCompleteBase() {
     http_parser_pause(&parser_, 1);
     return;
   }
-  onMessageComplete();
+
+  // If true, this indicates we were processing trailers and must
+  // move the last header into current_header_map_
+  if (header_parsing_state_ == HeaderParsingState::Value) {
+    completeLastHeader();
+  }
+
+  onMessageComplete(std::move(current_header_map_));
 }
 
 void ConnectionImpl::onMessageBeginBase() {
@@ -586,10 +637,11 @@ void ConnectionImpl::onResetStreamBase(StreamResetReason reason) {
 
 ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection, Stats::Scope& stats,
                                            ServerConnectionCallbacks& callbacks,
-                                           Http1Settings settings, uint32_t max_request_headers_kb,
+                                           const Http1Settings& settings,
+                                           uint32_t max_request_headers_kb,
                                            const uint32_t max_request_headers_count)
     : ConnectionImpl(connection, stats, HTTP_REQUEST, max_request_headers_kb,
-                     max_request_headers_count, formatter(settings)),
+                     max_request_headers_count, formatter(settings), settings.enable_trailers_),
       callbacks_(callbacks), codec_settings_(settings) {}
 
 void ServerConnectionImpl::onEncodeComplete() {
@@ -646,6 +698,8 @@ void ServerConnectionImpl::handlePath(HeaderMapImpl& headers, unsigned int metho
 }
 
 int ServerConnectionImpl::onHeadersComplete(HeaderMapImplPtr&& headers) {
+  ENVOY_CONN_LOG(trace, "Server: onHeadersComplete size={}", connection_, headers->size());
+
   // Handle the case where response happens prior to request complete. It's up to upper layer code
   // to disconnect the connection but we shouldn't fire any more events since it doesn't make
   // sense.
@@ -710,16 +764,17 @@ void ServerConnectionImpl::onBody(const char* data, size_t length) {
   }
 }
 
-void ServerConnectionImpl::onMessageComplete() {
+void ServerConnectionImpl::onMessageComplete(HeaderMapImplPtr&& trailers) {
   if (active_request_) {
-    Buffer::OwnedImpl buffer;
     active_request_->remote_complete_ = true;
-
     if (deferred_end_stream_headers_) {
       active_request_->request_decoder_->decodeHeaders(std::move(deferred_end_stream_headers_),
                                                        true);
       deferred_end_stream_headers_.reset();
+    } else if (processing_trailers_) {
+      active_request_->request_decoder_->decodeTrailers(std::move(trailers));
     } else {
+      Buffer::OwnedImpl buffer;
       active_request_->request_decoder_->decodeData(buffer, true);
     }
   }
@@ -765,7 +820,7 @@ ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, Stat
                                            ConnectionCallbacks&, const Http1Settings& settings,
                                            const uint32_t max_response_headers_count)
     : ConnectionImpl(connection, stats, HTTP_RESPONSE, MAX_RESPONSE_HEADERS_KB,
-                     max_response_headers_count, formatter(settings)) {}
+                     max_response_headers_count, formatter(settings), settings.enable_trailers_) {}
 
 bool ClientConnectionImpl::cannotHaveBody() {
   if ((!pending_responses_.empty() && pending_responses_.front().head_request_) ||
@@ -798,6 +853,7 @@ void ClientConnectionImpl::onEncodeHeaders(const HeaderMap& headers) {
 }
 
 int ClientConnectionImpl::onHeadersComplete(HeaderMapImplPtr&& headers) {
+  ENVOY_CONN_LOG(trace, "Client: onHeadersComplete size={}", connection_, headers->size());
   headers->setStatus(parser_.status_code);
 
   // Handle the case where the client is closing a kept alive connection (by sending a 408
@@ -832,7 +888,7 @@ void ClientConnectionImpl::onBody(const char* data, size_t length) {
   }
 }
 
-void ClientConnectionImpl::onMessageComplete() {
+void ClientConnectionImpl::onMessageComplete(HeaderMapImplPtr&& trailers) {
   ENVOY_CONN_LOG(trace, "message complete", connection_);
   if (ignore_message_complete_for_100_continue_) {
     ignore_message_complete_for_100_continue_ = false;
@@ -858,6 +914,8 @@ void ClientConnectionImpl::onMessageComplete() {
     if (deferred_end_stream_headers_) {
       response.decoder_->decodeHeaders(std::move(deferred_end_stream_headers_), true);
       deferred_end_stream_headers_.reset();
+    } else if (processing_trailers_) {
+      response.decoder_->decodeTrailers(std::move(trailers));
     } else {
       Buffer::OwnedImpl buffer;
       response.decoder_->decodeData(buffer, true);

source/common/http/http1/codec_impl.h

@@ -192,10 +192,12 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable<Log
 
   CodecStats& stats() { return stats_; }
 
+  const bool enable_trailers_ : 1;
+
 protected:
   ConnectionImpl(Network::Connection& connection, Stats::Scope& stats, http_parser_type type,
                  uint32_t max_headers_kb, const uint32_t max_headers_count,
-                 HeaderKeyFormatterPtr&& header_key_formatter);
+                 HeaderKeyFormatterPtr&& header_key_formatter, bool enable_trailers);
 
   bool resetStreamCalled() { return reset_stream_called_; }
 
@@ -204,6 +206,7 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable<Log
   http_parser parser_;
   HeaderMapPtr deferred_end_stream_headers_;
   Http::Code error_code_{Http::Code::BadRequest};
+  bool processing_trailers_{};
   bool handling_upgrade_{};
   const HeaderKeyFormatterPtr header_key_formatter_;
 
@@ -269,7 +272,7 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable<Log
    * Called when the request/response is complete.
    */
   void onMessageCompleteBase();
-  virtual void onMessageComplete() PURE;
+  virtual void onMessageComplete(HeaderMapImplPtr&& trailers) PURE;
 
   /**
    * @see onResetStreamBase().
@@ -318,7 +321,7 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable<Log
 class ServerConnectionImpl : public ServerConnection, public ConnectionImpl {
 public:
   ServerConnectionImpl(Network::Connection& connection, Stats::Scope& stats,
-                       ServerConnectionCallbacks& callbacks, Http1Settings settings,
+                       ServerConnectionCallbacks& callbacks, const Http1Settings& settings,
                        uint32_t max_request_headers_kb, const uint32_t max_request_headers_count);
 
   bool supports_http_10() override { return codec_settings_.accept_http_10_; }
@@ -354,7 +357,7 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl {
   void onUrl(const char* data, size_t length) override;
   int onHeadersComplete(HeaderMapImplPtr&& headers) override;
   void onBody(const char* data, size_t length) override;
-  void onMessageComplete() override;
+  void onMessageComplete(HeaderMapImplPtr&& trailers) override;
   void onResetStream(StreamResetReason reason) override;
   void sendProtocolError() override;
   void onAboveHighWatermark() override;
@@ -394,7 +397,7 @@ class ClientConnectionImpl : public ClientConnection, public ConnectionImpl {
   void onUrl(const char*, size_t) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; }
   int onHeadersComplete(HeaderMapImplPtr&& headers) override;
   void onBody(const char* data, size_t length) override;
-  void onMessageComplete() override;
+  void onMessageComplete(HeaderMapImplPtr&& trailers) override;
   void onResetStream(StreamResetReason reason) override;
   void sendProtocolError() override {}
   void onAboveHighWatermark() override;

source/common/http/utility.cc

@@ -277,6 +277,7 @@ Utility::parseHttp1Settings(const envoy::api::v2::core::Http1ProtocolOptions& co
   ret.allow_absolute_url_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, allow_absolute_url, true);
   ret.accept_http_10_ = config.accept_http_10();
   ret.default_host_for_http_10_ = config.default_host_for_http_10();
+  ret.enable_trailers_ = config.enable_trailers();
 
   if (config.header_key_format().has_proper_case_words()) {
     ret.header_key_format_ = Http1Settings::HeaderKeyFormat::ProperCase;