http2: limit the number of inbound frames. (envoyproxy#20)

This change adds protections against flooding using PRIORITY
and/or WINDOW_UPDATE frames, as well as frames with an empty
payload and no end stream flag.

Fixes CVE-2019-9511, CVE-2019-9513 and CVE-2019-9518.

Signed-off-by: Piotr Sikora <[email protected]>

Author: PiotrSikora

api/envoy/api/v2/core/protocol.proto

@@ -49,6 +49,7 @@ message Http1ProtocolOptions {
   string default_host_for_http_10 = 3;
 }
 
+// [#comment:next free field: 12]
 message Http2ProtocolOptions {
   // `Maximum table size <https://httpwg.org/specs/rfc7541.html#rfc.section.4.2>`_
   // (in octets) that the encoder is permitted to use for the dynamic HPACK table. Valid values
@@ -94,18 +95,53 @@ message Http2ProtocolOptions {
 
   // Limit the number of pending outbound downstream frames of all types (frames that are waiting to
   // be written into the socket). Exceeding this limit triggers flood mitigation and connection is
-  // terminated. The "http2.outbound_flood" stat tracks the number of terminated connections due to
-  // flood mitigation. The default limit is 10000.
+  // terminated. The ``http2.outbound_flood`` stat tracks the number of terminated connections due
+  // to flood mitigation. The default limit is 10000.
   // [#comment:TODO: implement same limits for upstream outbound frames as well.]
   google.protobuf.UInt32Value max_outbound_frames = 7 [(validate.rules).uint32 = {gte: 1}];
 
   // Limit the number of pending outbound downstream frames of types PING, SETTINGS and RST_STREAM,
   // preventing high memory utilization when receiving continuous stream of these frames. Exceeding
   // this limit triggers flood mitigation and connection is terminated. The
-  // "http2.outbound_control_flood" stat tracks the number of terminated connections due to flood
+  // ``http2.outbound_control_flood`` stat tracks the number of terminated connections due to flood
   // mitigation. The default limit is 1000.
   // [#comment:TODO: implement same limits for upstream outbound frames as well.]
   google.protobuf.UInt32Value max_outbound_control_frames = 8 [(validate.rules).uint32 = {gte: 1}];
+
+  // Limit the number of consecutive inbound frames of types HEADERS, CONTINUATION and DATA with an
+  // empty payload and no end stream flag. Those frames have no legitimate use and are abusive, but
+  // might be a result of a broken HTTP/2 implementation. The `http2.inbound_empty_frames_flood``
+  // stat tracks the number of connections terminated due to flood mitigation.
+  // Setting this to 0 will terminate connection upon receiving first frame with an empty payload
+  // and no end stream flag. The default limit is 1.
+  // [#comment:TODO: implement same limits for upstream inbound frames as well.]
+  google.protobuf.UInt32Value max_consecutive_inbound_frames_with_empty_payload = 9;
+
+  // Limit the number of inbound PRIORITY frames allowed per each opened stream. If the number
+  // of PRIORITY frames received over the lifetime of connection exceeds the value calculated
+  // using this formula::
+  //
+  //     max_inbound_priority_frames_per_stream * (1 + inbound_streams)
+  //
+  // the connection is terminated. The ``http2.inbound_priority_frames_flood`` stat tracks
+  // the number of connections terminated due to flood mitigation. The default limit is 100.
+  // [#comment:TODO: implement same limits for upstream inbound frames as well.]
+  google.protobuf.UInt32Value max_inbound_priority_frames_per_stream = 10;
+
+  // Limit the number of inbound WINDOW_UPDATE frames allowed per DATA frame sent. If the number
+  // of WINDOW_UPDATE frames received over the lifetime of connection exceeds the value calculated
+  // using this formula::
+  //
+  //     1 + 2 * (inbound_streams +
+  //              max_inbound_window_update_frames_per_data_frame_sent * outbound_data_frames)
+  //
+  // the connection is terminated. The ``http2.inbound_priority_frames_flood`` stat tracks
+  // the number of connections terminated due to flood mitigation. The default limit is 10.
+  // Setting this to 1 should be enough to support HTTP/2 implementations with basic flow control,
+  // but more complex implementations that try to estimate available bandwidth require at least 2.
+  // [#comment:TODO: implement same limits for upstream inbound frames as well.]
+  google.protobuf.UInt32Value max_inbound_window_update_frames_per_data_frame_sent = 11
+      [(validate.rules).uint32 = {gte: 1}];
 }
 
 // [#not-implemented-hide:]

docs/root/configuration/http_conn_man/stats.rst

@@ -111,6 +111,9 @@ All http2 statistics are rooted at *http2.*
 
    header_overflow, Counter, Total number of connections reset due to the headers being larger than the :ref:`configured value <envoy_api_field_config.filter.network.http_connection_manager.v2.HttpConnectionManager.max_request_headers_kb>`.
    headers_cb_no_stream, Counter, Total number of errors where a header callback is called without an associated stream. This tracks an unexpected occurrence due to an as yet undiagnosed bug
+   inbound_empty_frames_flood, Counter, Total number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting <envoy_api_field_core.Http2ProtocolOptions.max_consecutive_inbound_frames_with_empty_payload>`.
+   inbound_priority_frames_flood, Counter, Total number of connections terminated for exceeding the limit on inbound frames of type PRIORITY. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting <envoy_api_field_core.Http2ProtocolOptions.max_inbound_priority_frames_per_stream>`.
+   inbound_window_update_frames_flood, Counter, Total number of connections terminated for exceeding the limit on inbound frames of type WINDOW_UPDATE. The limit is configured by setting the :ref:`max_inbound_window_updateframes_per_data_frame_sent config setting <envoy_api_field_core.Http2ProtocolOptions.max_inbound_window_update_frames_per_data_frame_sent>`.
    outbound_flood, Counter, Total number of connections terminated for exceeding the limit on outbound frames of all types. The limit is configured by setting the :ref:`max_outbound_frames config setting <envoy_api_field_core.Http2ProtocolOptions.max_outbound_frames>`.
    outbound_control_flood, Counter, "Total number of connections terminated for exceeding the limit on outbound frames of types PING, SETTINGS and RST_STREAM. The limit is configured by setting the :ref:`max_outbound_control_frames config setting <envoy_api_field_core.Http2ProtocolOptions.max_outbound_control_frames>`."
    rx_messaging_error, Counter, Total number of invalid received frames that violated `section 8 <https://tools.ietf.org/html/rfc7540#section-8>`_ of the HTTP/2 spec. This will result in a *tx_reset*

docs/root/intro/version_history.rst

@@ -5,7 +5,10 @@ Version history
 ================
 1.11.1 (Pending)
 ================
-* http: added mitigation of client initiated atacks that result in flooding of the outbound queue of downstream HTTP/2 connections.
+* http: added mitigation of client initiated atacks that result in flooding of the downstream HTTP/2 connections.
+* http: added :ref:`inbound_empty_frames_flood <config_http_conn_man_stats_per_codec>` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting <envoy_api_field_core.Http2ProtocolOptions.max_consecutive_inbound_frames_with_empty_payload>`.
+* http: added :ref:`inbound_priority_frames_flood <config_http_conn_man_stats_per_codec>` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound PRIORITY frames. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting <envoy_api_field_core.Http2ProtocolOptions.max_inbound_priority_frames_per_stream>`.
+* http: added :ref:`inbound_window_update_frames_flood <config_http_conn_man_stats_per_codec>` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound WINDOW_UPDATE frames. The limit is configured by setting the :ref:`max_inbound_window_update_frames_per_data_frame_sent config setting <envoy_api_field_core.Http2ProtocolOptions.max_inbound_window_update_frames_per_data_frame_sent>`.
 * http: added :ref:`outbound_flood <config_http_conn_man_stats_per_codec>` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit. The limit is configured by setting the :ref:`max_outbound_frames config setting <envoy_api_field_core.Http2ProtocolOptions.max_outbound_frames>`
 * http: added :ref:`outbound_control_flood <config_http_conn_man_stats_per_codec>` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit for PING, SETTINGS and RST_STREAM frames. The limit is configured by setting the :ref:`max_outbound_control_frames config setting <envoy_api_field_core.Http2ProtocolOptions.max_outbound_control_frames>`.
 

include/envoy/http/codec.h

@@ -237,6 +237,11 @@ struct Http2Settings {
   bool allow_metadata_{DEFAULT_ALLOW_METADATA};
   uint32_t max_outbound_frames_{DEFAULT_MAX_OUTBOUND_FRAMES};
   uint32_t max_outbound_control_frames_{DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES};
+  uint32_t max_consecutive_inbound_frames_with_empty_payload_{
+      DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD};
+  uint32_t max_inbound_priority_frames_per_stream_{DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM};
+  uint32_t max_inbound_window_update_frames_per_data_frame_sent_{
+      DEFAULT_MAX_INBOUND_WINDOW_UPDATE_FRAMES_PER_DATA_FRAME_SENT};
 
   // disable HPACK compression
   static const uint32_t MIN_HPACK_TABLE_SIZE = 0;
@@ -279,6 +284,13 @@ struct Http2Settings {
   static const uint32_t DEFAULT_MAX_OUTBOUND_FRAMES = 10000;
   // Default limit on the number of outbound frames of types PING, SETTINGS and RST_STREAM.
   static const uint32_t DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES = 1000;
+  // Default limit on the number of consecutive inbound frames with an empty payload
+  // and no end stream flag.
+  static const uint32_t DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD = 1;
+  // Default limit on the number of inbound frames of type PRIORITY (per stream).
+  static const uint32_t DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM = 100;
+  // Default limit on the number of inbound frames of type WINDOW_UPDATE (per DATA frame sent).
+  static const uint32_t DEFAULT_MAX_INBOUND_WINDOW_UPDATE_FRAMES_PER_DATA_FRAME_SENT = 10;
 };
 
 /**

source/common/http/http2/codec_impl.cc

@@ -252,6 +252,8 @@ int ConnectionImpl::StreamImpl::onDataSourceSend(const uint8_t* framehd, size_t
   // https://nghttp2.org/documentation/types.html#c.nghttp2_send_data_callback
   static const uint64_t FRAME_HEADER_SIZE = 9;
 
+  parent_.outbound_data_frames_++;
+
   Buffer::OwnedImpl output;
   if (!parent_.addOutboundFrameFragment(output, framehd, FRAME_HEADER_SIZE)) {
     ENVOY_CONN_LOG(debug, "error sending data frame: Too many frames in the outbound queue",
@@ -355,7 +357,7 @@ void ConnectionImpl::dispatch(Buffer::Instance& data) {
     dispatching_ = true;
     ssize_t rc =
         nghttp2_session_mem_recv(session_, static_cast<const uint8_t*>(slice.mem_), slice.len_);
-    if (rc == NGHTTP2_ERR_FLOODED) {
+    if (rc == NGHTTP2_ERR_FLOODED || flood_detected_) {
       throw FrameFloodException(
           "Flooding was detected in this HTTP/2 session, and it must be closed");
     }
@@ -408,9 +410,36 @@ void ConnectionImpl::shutdownNotice() {
   sendPendingFrames();
 }
 
+int ConnectionImpl::onBeforeFrameReceived(const nghttp2_frame_hd* hd) {
+  ENVOY_CONN_LOG(trace, "about to recv frame type={}, flags={}", connection_,
+                 static_cast<uint64_t>(hd->type), static_cast<uint64_t>(hd->flags));
+
+  // Track all the frames without padding here, since this is the only callback we receive
+  // for some of them (e.g. CONTINUATION frame, frames sent on closed streams, etc.).
+  // HEADERS frame is tracked in onBeginHeaders(), DATA frame is tracked in onFrameReceived().
+  if (hd->type != NGHTTP2_HEADERS && hd->type != NGHTTP2_DATA) {
+    if (!trackInboundFrames(hd, 0)) {
+      return NGHTTP2_ERR_FLOODED;
+    }
+  }
+
+  return 0;
+}
+
 int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) {
   ENVOY_CONN_LOG(trace, "recv frame type={}", connection_, static_cast<uint64_t>(frame->hd.type));
 
+  // onFrameReceived() is called with a complete HEADERS frame assembled from all the HEADERS
+  // and CONTINUATION frames, but we track them separately: HEADERS frames in onBeginHeaders()
+  // and CONTINUATION frames in onBeforeFrameReceived().
+  ASSERT(frame->hd.type != NGHTTP2_CONTINUATION);
+
+  if (frame->hd.type == NGHTTP2_DATA) {
+    if (!trackInboundFrames(&frame->hd, frame->data.padlen)) {
+      return NGHTTP2_ERR_FLOODED;
+    }
+  }
+
   // Only raise GOAWAY once, since we don't currently expose stream information. Shutdown
   // notifications are the same as a normal GOAWAY.
   if (frame->hd.type == NGHTTP2_GOAWAY && !raised_goaway_) {
@@ -567,7 +596,7 @@ int ConnectionImpl::onInvalidFrame(int32_t stream_id, int error_code) {
 }
 
 int ConnectionImpl::onBeforeFrameSend(const nghttp2_frame* frame) {
-  ENVOY_CONN_LOG(trace, "about to sent frame type={}, flags={}", connection_,
+  ENVOY_CONN_LOG(trace, "about to send frame type={}, flags={}", connection_,
                  static_cast<uint64_t>(frame->hd.type), static_cast<uint64_t>(frame->hd.flags));
   ASSERT(!is_outbound_flood_monitored_control_frame_);
   // Flag flood monitored outbound control frames.
@@ -882,6 +911,11 @@ ConnectionImpl::Http2Callbacks::Http2Callbacks() {
         return static_cast<ConnectionImpl*>(user_data)->onData(stream_id, data, len);
       });
 
+  nghttp2_session_callbacks_set_on_begin_frame_callback(
+      callbacks_, [](nghttp2_session*, const nghttp2_frame_hd* hd, void* user_data) -> int {
+        return static_cast<ConnectionImpl*>(user_data)->onBeforeFrameReceived(hd);
+      });
+
   nghttp2_session_callbacks_set_on_frame_recv_callback(
       callbacks_, [](nghttp2_session*, const nghttp2_frame* frame, void* user_data) -> int {
         return static_cast<ConnectionImpl*>(user_data)->onFrameReceived(frame);
@@ -1042,6 +1076,11 @@ ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection,
 int ServerConnectionImpl::onBeginHeaders(const nghttp2_frame* frame) {
   // For a server connection, we should never get push promise frames.
   ASSERT(frame->hd.type == NGHTTP2_HEADERS);
+
+  if (!trackInboundFrames(&frame->hd, frame->headers.padlen)) {
+    return NGHTTP2_ERR_FLOODED;
+  }
+
   if (frame->headers.cat != NGHTTP2_HCAT_REQUEST) {
     stats_.trailers_.inc();
     ASSERT(frame->headers.cat == NGHTTP2_HCAT_HEADERS);
@@ -1072,6 +1111,82 @@ int ServerConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& na
   return saveHeader(frame, std::move(name), std::move(value));
 }
 
+bool ServerConnectionImpl::trackInboundFrames(const nghttp2_frame_hd* hd, uint32_t padding_length) {
+  ENVOY_CONN_LOG(trace, "track inbound frame type={} flags={} length={} padding_length={}",
+                 connection_, static_cast<uint64_t>(hd->type), static_cast<uint64_t>(hd->flags),
+                 static_cast<uint64_t>(hd->length), padding_length);
+  switch (hd->type) {
+  case NGHTTP2_HEADERS:
+  case NGHTTP2_CONTINUATION:
+    // Track new streams.
+    if (hd->flags & NGHTTP2_FLAG_END_HEADERS) {
+      inbound_streams_++;
+    }
+    FALLTHRU;
+  case NGHTTP2_DATA:
+    // Track frames with an empty payload and no end stream flag.
+    if (hd->length - padding_length == 0 && !(hd->flags & NGHTTP2_FLAG_END_STREAM)) {
+      ENVOY_CONN_LOG(trace, "frame with an empty payload and no end stream flag.", connection_);
+      consecutive_inbound_frames_with_empty_payload_++;
+    } else {
+      consecutive_inbound_frames_with_empty_payload_ = 0;
+    }
+    break;
+  case NGHTTP2_PRIORITY:
+    inbound_priority_frames_++;
+    break;
+  case NGHTTP2_WINDOW_UPDATE:
+    inbound_window_update_frames_++;
+    break;
+  default:
+    break;
+  }
+
+  if (!checkInboundFrameLimits()) {
+    // NGHTTP2_ERR_FLOODED is overridden within nghttp2 library and it doesn't propagate
+    // all the way to nghttp2_session_mem_recv() where we need it.
+    flood_detected_ = true;
+    return false;
+  }
+
+  return true;
+}
+
+bool ServerConnectionImpl::checkInboundFrameLimits() {
+  ASSERT(dispatching_downstream_data_);
+
+  if (consecutive_inbound_frames_with_empty_payload_ >
+      max_consecutive_inbound_frames_with_empty_payload_) {
+    ENVOY_CONN_LOG(trace,
+                   "error reading frame: Too many consecutive frames with an empty payload "
+                   "received in this HTTP/2 session.",
+                   connection_);
+    stats_.inbound_empty_frames_flood_.inc();
+    return false;
+  }
+
+  if (inbound_priority_frames_ > max_inbound_priority_frames_per_stream_ * (1 + inbound_streams_)) {
+    ENVOY_CONN_LOG(trace,
+                   "error reading frame: Too many PRIORITY frames received in this HTTP/2 session.",
+                   connection_);
+    stats_.inbound_priority_frames_flood_.inc();
+    return false;
+  }
+
+  if (inbound_window_update_frames_ >
+      1 + 2 * (inbound_streams_ +
+               max_inbound_window_update_frames_per_data_frame_sent_ * outbound_data_frames_)) {
+    ENVOY_CONN_LOG(
+        trace,
+        "error reading frame: Too many WINDOW_UPDATE frames received in this HTTP/2 session.",
+        connection_);
+    stats_.inbound_window_update_frames_flood_.inc();
+    return false;
+  }
+
+  return true;
+}
+
 void ServerConnectionImpl::checkOutboundQueueLimits() {
   if (outbound_frames_ > max_outbound_frames_ && dispatching_downstream_data_) {
     stats_.outbound_flood_.inc();