[Bandwidth-Limit] Add response trailers for bandwidth delay

api/envoy/extensions/filters/http/bandwidth_limit/v3/bandwidth_limit.proto

@@ -19,7 +19,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // Bandwidth limit :ref:`configuration overview <config_http_filters_bandwidth_limit>`.
 // [#extension: envoy.filters.http.bandwidth_limit]
 
-// [#next-free-field: 6]
+// [#next-free-field: 8]
 message BandwidthLimit {
   // Defines the mode for the bandwidth limit filter.
   // Values represent bitmask.
@@ -66,4 +66,19 @@ message BandwidthLimit {
   // Runtime flag that controls whether the filter is enabled or not. If not specified, defaults
   // to enabled.
   config.core.v3.RuntimeFeatureFlag runtime_enabled = 5;
+
+  // Enable response trailers.
+  //
+  // .. note::
+  //
+  //   * If set true, the response trailers *bandwidth-request-delay-ms* and *bandwidth-response-delay-ms* will be added, prefixed by *response_trailer_prefix*.
+  //   * bandwidth-request-delay-ms: delay time in milliseconds it took for the request stream transfer.
+  //   * bandwidth-response-delay-ms: delay time in milliseconds it took for the response stream transfer.
+  //   * If :ref:`enable_mode <envoy_v3_api_field_extensions.filters.http.bandwidth_limit.v3.BandwidthLimit.enable_mode>` is DISABLED or REQUEST, the trailers will not be set.
+  //   * If both the request and response delay time is 0, the trailers will not be set.
+  //
+  bool enable_response_trailers = 6;
+
+  // Optional The prefix for the response trailers.
+  string response_trailer_prefix = 7;
 }

docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst

@@ -42,14 +42,16 @@ The HTTP bandwidth limit filter outputs statistics in the ``<stat_prefix>.http_b
   :widths: 1, 1, 2
 
   request_enabled, Counter, Total number of request streams for which the bandwidth limiter was consulted
+  request_enforced, Counter, Total number of request streams for which the bandwidth limiter was enforced
   request_pending, GAUGE, Number of request streams which are currently pending transfer in bandwidth limiter
-  request_incoming_size, GAUGE, Size in bytes of incoming request data to bandwidth limiter
-  request_allowed_size, GAUGE, Size in bytes of outgoing request data from bandwidth limiter
+  request_incoming_size, Counter, Size in bytes of incoming request data to bandwidth limiter
+  request_allowed_size, Counter, Size in bytes of outgoing request data from bandwidth limiter
   request_transfer_duration, HISTOGRAM, Total time (including added delay) it took for the request stream transfer
   response_enabled, Counter, Total number of response streams for which the bandwidth limiter was consulted
+  response_enforced, Counter, Total number of response streams for which the bandwidth limiter was enforced
   response_pending, GAUGE, Number of response streams which are currently pending transfer in bandwidth limiter
-  response_incoming_size, GAUGE, Size in bytes of incoming response data to bandwidth limiter
-  response_allowed_size, GAUGE, Size in bytes of outgoing response data from bandwidth limiter
+  response_incoming_size, Counter, Size in bytes of incoming response data to bandwidth limiter
+  response_allowed_size, Counter, Size in bytes of outgoing response data from bandwidth limiter
   response_transfer_duration, HISTOGRAM, Total time (including added delay) it took for the response stream transfer
 
 .. _config_http_filters_bandwidth_limit_runtime:

docs/root/version_history/current.rst

@@ -9,6 +9,8 @@ Minor Behavior Changes
 ----------------------
 *Changes that may cause incompatibilities for some users, but should not for most*
 
+* bandwidth_limit: added :ref:`response trailers <envoy_v3_api_field_extensions.filters.http.bandwidth_limit.v3.BandwidthLimit.enable_response_trailers>` when request or response delay are enforced.
+* bandwidth_limit: added :ref:`bandwidth limit stats <config_http_filters_bandwidth_limit>` *request_enforced* and *response_enforced*.
 * config: the log message for "gRPC config stream closed" now uses the most recent error message, and reports seconds instead of milliseconds for how long the most recent status has been received.
 * dns: now respecting the returned DNS TTL for resolved hosts, rather than always relying on the hard-coded :ref:`dns_refresh_rate. <envoy_v3_api_field_config.cluster.v3.Cluster.dns_refresh_rate>` This behavior can be temporarily reverted by setting the runtime guard ``envoy.reloadable_features.use_dns_ttl`` to false.
 * listener: destroy per network filter chain stats when a network filter chain is removed during the listener in place update.

source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc

@@ -16,14 +16,30 @@ namespace Extensions {
 namespace HttpFilters {
 namespace BandwidthLimitFilter {
 
+namespace {
+const Http::LowerCaseString DefaultRequestDelayTrailer =
+    Http::LowerCaseString("bandwidth-request-delay-ms");
+const Http::LowerCaseString DefaultResponseDelayTrailer =
+    Http::LowerCaseString("bandwidth-response-delay-ms");
+} // namespace
+
 FilterConfig::FilterConfig(const BandwidthLimit& config, Stats::Scope& scope,
                            Runtime::Loader& runtime, TimeSource& time_source, bool per_route)
     : runtime_(runtime), time_source_(time_source), enable_mode_(config.enable_mode()),
       limit_kbps_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, limit_kbps, 0)),
       fill_interval_(std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(
           config, fill_interval, StreamRateLimiter::DefaultFillInterval.count()))),
       enabled_(config.runtime_enabled(), runtime),
-      stats_(generateStats(config.stat_prefix(), scope)) {
+      stats_(generateStats(config.stat_prefix(), scope)),
+      request_delay_trailer_(config.response_trailer_prefix().empty()
+                                 ? DefaultRequestDelayTrailer
+                                 : Http::LowerCaseString(config.response_trailer_prefix() + "-" +
+                                                         DefaultRequestDelayTrailer.get())),
+      response_delay_trailer_(config.response_trailer_prefix().empty()
+                                  ? DefaultResponseDelayTrailer
+                                  : Http::LowerCaseString(config.response_trailer_prefix() + "-" +
+                                                          DefaultResponseDelayTrailer.get())),
+      enable_response_trailers_(config.enable_response_trailers()) {
   if (per_route && !config.has_limit_kbps()) {
     throw EnvoyException("bandwidthlimitfilter: limit must be set for per route filter config");
   }
@@ -64,7 +80,12 @@ Http::FilterHeadersStatus BandwidthLimiter::decodeHeaders(Http::RequestHeaderMap
           updateStatsOnDecodeFinish();
           decoder_callbacks_->continueDecoding();
         },
-        [config](uint64_t len) { config.stats().request_allowed_size_.set(len); },
+        [&config](uint64_t len, bool limit_enforced) {
+          config.stats().request_allowed_size_.add(len);
+          if (limit_enforced) {
+            config.stats().request_enforced_.inc();
+          }
+        },
         const_cast<FilterConfig*>(&config)->timeSource(), decoder_callbacks_->dispatcher(),
         decoder_callbacks_->scope(), config.tokenBucket(), config.fillInterval());
   }
@@ -82,7 +103,7 @@ Http::FilterDataStatus BandwidthLimiter::decodeData(Buffer::Instance& data, bool
           const_cast<FilterConfig*>(&config)->timeSource());
       config.stats().request_pending_.inc();
     }
-    config.stats().request_incoming_size_.set(data.length());
+    config.stats().request_incoming_size_.add(data.length());
 
     request_limiter_->writeData(data, end_stream);
     return Http::FilterDataStatus::StopIterationNoBuffer;
@@ -123,7 +144,12 @@ Http::FilterHeadersStatus BandwidthLimiter::encodeHeaders(Http::ResponseHeaderMa
           updateStatsOnEncodeFinish();
           encoder_callbacks_->continueEncoding();
         },
-        [config](uint64_t len) { config.stats().response_allowed_size_.set(len); },
+        [&config](uint64_t len, bool limit_enforced) {
+          config.stats().response_allowed_size_.add(len);
+          if (limit_enforced) {
+            config.stats().response_enforced_.inc();
+          }
+        },
         const_cast<FilterConfig*>(&config)->timeSource(), encoder_callbacks_->dispatcher(),
         encoder_callbacks_->scope(), config.tokenBucket(), config.fillInterval());
   }
@@ -135,23 +161,34 @@ Http::FilterDataStatus BandwidthLimiter::encodeData(Buffer::Instance& data, bool
   if (response_limiter_ != nullptr) {
     const auto& config = getConfig();
 
+    // Adds encoded trailers. May only be called in encodeData when end_stream is set to true.
+    // If upstream has trailers, addEncodedTrailers won't be called
+    bool trailer_added = false;
+    if (end_stream) {
+      trailers_ = &encoder_callbacks_->addEncodedTrailers();
+      trailer_added = true;
+    }
+
     if (!response_latency_) {
       response_latency_ = std::make_unique<Stats::HistogramCompletableTimespanImpl>(
           config.stats().response_transfer_duration_,
           const_cast<FilterConfig*>(&config)->timeSource());
       config.stats().response_pending_.inc();
     }
-    config.stats().response_incoming_size_.set(data.length());
+    config.stats().response_incoming_size_.add(data.length());
 
-    response_limiter_->writeData(data, end_stream);
+    response_limiter_->writeData(data, end_stream, trailer_added);
     return Http::FilterDataStatus::StopIterationNoBuffer;
   }
   ENVOY_LOG(debug, "BandwidthLimiter <encode data>: response_limiter not set");
   return Http::FilterDataStatus::Continue;
 }
 
-Http::FilterTrailersStatus BandwidthLimiter::encodeTrailers(Http::ResponseTrailerMap&) {
+Http::FilterTrailersStatus
+BandwidthLimiter::encodeTrailers(Http::ResponseTrailerMap& response_trailers) {
   if (response_limiter_ != nullptr) {
+    trailers_ = &response_trailers;
+
     if (response_limiter_->onTrailers()) {
       return Http::FilterTrailersStatus::StopIteration;
     } else {
@@ -164,6 +201,7 @@ Http::FilterTrailersStatus BandwidthLimiter::encodeTrailers(Http::ResponseTraile
 
 void BandwidthLimiter::updateStatsOnDecodeFinish() {
   if (request_latency_) {
+    request_duration_ = request_latency_.get()->elapsed().count();
     request_latency_->complete();
     request_latency_.reset();
     getConfig().stats().request_pending_.dec();
@@ -172,9 +210,21 @@ void BandwidthLimiter::updateStatsOnDecodeFinish() {
 
 void BandwidthLimiter::updateStatsOnEncodeFinish() {
   if (response_latency_) {
+    const auto& config = getConfig();
+
+    if (config.enableResponseTrailers() && trailers_ != nullptr) {
+      auto response_duration = response_latency_.get()->elapsed().count();
+      if (request_duration_ > 0) {
+        trailers_->setCopy(config.requestDelayTrailer(), std::to_string(request_duration_));
+      }
+      if (response_duration > 0) {
+        trailers_->setCopy(config.responseDelayTrailer(), std::to_string(response_duration));
+      }
+    }
+
     response_latency_->complete();
     response_latency_.reset();
-    getConfig().stats().response_pending_.dec();
+    config.stats().response_pending_.dec();
   }
 }
 

source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h

@@ -32,12 +32,14 @@ namespace BandwidthLimitFilter {
 #define ALL_BANDWIDTH_LIMIT_STATS(COUNTER, GAUGE, HISTOGRAM)                                       \
   COUNTER(request_enabled)                                                                         \
   COUNTER(response_enabled)                                                                        \
+  COUNTER(request_enforced)                                                                        \
+  COUNTER(response_enforced)                                                                       \
   GAUGE(request_pending, Accumulate)                                                               \
   GAUGE(response_pending, Accumulate)                                                              \
-  GAUGE(request_incoming_size, Accumulate)                                                         \
-  GAUGE(response_incoming_size, Accumulate)                                                        \
-  GAUGE(request_allowed_size, Accumulate)                                                          \
-  GAUGE(response_allowed_size, Accumulate)                                                         \
+  COUNTER(request_incoming_size)                                                                   \
+  COUNTER(response_incoming_size)                                                                  \
+  COUNTER(request_allowed_size)                                                                    \
+  COUNTER(response_allowed_size)                                                                   \
   HISTOGRAM(request_transfer_duration, Milliseconds)                                               \
   HISTOGRAM(response_transfer_duration, Milliseconds)
 
@@ -70,6 +72,9 @@ class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig {
   EnableMode enableMode() const { return enable_mode_; };
   const std::shared_ptr<SharedTokenBucketImpl> tokenBucket() const { return token_bucket_; }
   std::chrono::milliseconds fillInterval() const { return fill_interval_; }
+  const Http::LowerCaseString& requestDelayTrailer() const { return request_delay_trailer_; }
+  const Http::LowerCaseString& responseDelayTrailer() const { return response_delay_trailer_; }
+  bool enableResponseTrailers() const { return enable_response_trailers_; }
 
 private:
   friend class FilterTest;
@@ -85,6 +90,9 @@ class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig {
   mutable BandwidthLimitStats stats_;
   // Filter chain's shared token bucket
   std::shared_ptr<SharedTokenBucketImpl> token_bucket_;
+  const Http::LowerCaseString request_delay_trailer_;
+  const Http::LowerCaseString response_delay_trailer_;
+  const bool enable_response_trailers_;
 };
 
 using FilterConfigSharedPtr = std::shared_ptr<FilterConfig>;
@@ -141,6 +149,8 @@ class BandwidthLimiter : public Http::StreamFilter, Logger::Loggable<Logger::Id:
   std::unique_ptr<Envoy::Extensions::HttpFilters::Common::StreamRateLimiter> response_limiter_;
   Stats::TimespanPtr request_latency_;
   Stats::TimespanPtr response_latency_;
+  uint64_t request_duration_ = 0;
+  Http::ResponseTrailerMap* trailers_;
 };
 
 } // namespace BandwidthLimitFilter

source/extensions/filters/http/common/stream_rate_limiter.cc

@@ -17,7 +17,7 @@ StreamRateLimiter::StreamRateLimiter(
     uint64_t max_kbps, uint64_t max_buffered_data, std::function<void()> pause_data_cb,
     std::function<void()> resume_data_cb,
     std::function<void(Buffer::Instance&, bool)> write_data_cb, std::function<void()> continue_cb,
-    std::function<void(uint64_t)> write_stats_cb, TimeSource& time_source,
+    std::function<void(uint64_t, bool)> write_stats_cb, TimeSource& time_source,
     Event::Dispatcher& dispatcher, const ScopeTrackedObject& scope,
     std::shared_ptr<TokenBucket> token_bucket, std::chrono::milliseconds fill_interval)
     : fill_interval_(std::move(fill_interval)), write_data_cb_(write_data_cb),
@@ -63,7 +63,7 @@ void StreamRateLimiter::onTokenTimer() {
   // Move the data to write into the output buffer with as little copying as possible.
   // NOTE: This might be moving zero bytes, but that should work fine.
   data_to_write.move(buffer_, bytes_to_write);
-  write_stats_cb_(bytes_to_write);
+  write_stats_cb_(bytes_to_write, buffer_.length() > 0);
 
   // If the buffer still contains data in it, we couldn't get enough tokens, so schedule the next
   // token available time.
@@ -88,10 +88,18 @@ void StreamRateLimiter::onTokenTimer() {
   }
 }
 
-void StreamRateLimiter::writeData(Buffer::Instance& incoming_buffer, bool end_stream) {
+void StreamRateLimiter::writeData(Buffer::Instance& incoming_buffer, bool end_stream,
+                                  bool trailer_added) {
   auto len = incoming_buffer.length();
   buffer_.move(incoming_buffer);
   saw_end_stream_ = end_stream;
+  // If trailer_added is true, set saw_trailers_ to true to continue encode trailers, added
+  // after buffer_.move to ensure buffer has data and won't invoke continue_cb_ before
+  // processing the data in last data frame.
+  if (trailer_added) {
+    saw_trailers_ = true;
+  }
+
   ENVOY_LOG(debug,
             "StreamRateLimiter <writeData>: got new {} bytes of data. token "
             "timer {} scheduled.",

source/extensions/filters/http/common/stream_rate_limiter.h

@@ -49,17 +49,17 @@ class StreamRateLimiter : Logger::Loggable<Logger::Id::filter> {
   StreamRateLimiter(uint64_t max_kbps, uint64_t max_buffered_data,
                     std::function<void()> pause_data_cb, std::function<void()> resume_data_cb,
                     std::function<void(Buffer::Instance&, bool)> write_data_cb,
-                    std::function<void()> continue_cb, std::function<void(uint64_t)> write_stats_cb,
-                    TimeSource& time_source, Event::Dispatcher& dispatcher,
-                    const ScopeTrackedObject& scope,
+                    std::function<void()> continue_cb,
+                    std::function<void(uint64_t, bool)> write_stats_cb, TimeSource& time_source,
+                    Event::Dispatcher& dispatcher, const ScopeTrackedObject& scope,
                     std::shared_ptr<TokenBucket> token_bucket = nullptr,
                     std::chrono::milliseconds fill_interval = DefaultFillInterval);
 
   /**
    * Called by the stream to write data. All data writes happen asynchronously, the stream should
    * be stopped after this call (all data will be drained from incoming_buffer).
    */
-  void writeData(Buffer::Instance& incoming_buffer, bool end_stream);
+  void writeData(Buffer::Instance& incoming_buffer, bool end_stream, bool trailer_added = false);
 
   /**
    * Called if the stream receives trailers.
@@ -83,7 +83,7 @@ class StreamRateLimiter : Logger::Loggable<Logger::Id::filter> {
   const std::chrono::milliseconds fill_interval_;
   const std::function<void(Buffer::Instance&, bool)> write_data_cb_;
   const std::function<void()> continue_cb_;
-  const std::function<void(uint64_t)> write_stats_cb_;
+  const std::function<void(uint64_t, bool)> write_stats_cb_;
   const ScopeTrackedObject& scope_;
   std::shared_ptr<TokenBucket> token_bucket_;
   Event::TimerPtr token_timer_;

source/extensions/filters/http/fault/fault_filter.cc

@@ -212,7 +212,7 @@ void FaultFilter::maybeSetupResponseRateLimit(const Http::RequestHeaderMap& requ
         encoder_callbacks_->injectEncodedDataToFilterChain(data, end_stream);
       },
       [this] { encoder_callbacks_->continueEncoding(); },
-      [](uint64_t) {
+      [](uint64_t, bool) {
         // write stats callback.
       },
       config_->timeSource(), decoder_callbacks_->dispatcher(), decoder_callbacks_->scope());

test/extensions/filters/http/bandwidth_limit/config_test.cc

@@ -37,6 +37,7 @@ TEST(Factory, RouteSpecificFilterConfig) {
   enable_mode: REQUEST_AND_RESPONSE
   limit_kbps: 10
   fill_interval: 0.1s
+  response_trailer_prefix: test
   )";
 
   BandwidthLimitFilterConfig factory;
@@ -53,6 +54,10 @@ TEST(Factory, RouteSpecificFilterConfig) {
   EXPECT_EQ(config->fillInterval().count(), 100);
   EXPECT_EQ(config->enableMode(), EnableMode::BandwidthLimit_EnableMode_REQUEST_AND_RESPONSE);
   EXPECT_FALSE(config->tokenBucket() == nullptr);
+  EXPECT_EQ(const_cast<FilterConfig*>(config)->requestDelayTrailer(),
+            Http::LowerCaseString("test-bandwidth-request-delay-ms"));
+  EXPECT_EQ(const_cast<FilterConfig*>(config)->responseDelayTrailer(),
+            Http::LowerCaseString("test-bandwidth-response-delay-ms"));
 }
 
 TEST(Factory, RouteSpecificFilterConfigDisabledByDefault) {
@@ -96,6 +101,11 @@ TEST(Factory, RouteSpecificFilterConfigDefaultFillInterval) {
   const auto* config = dynamic_cast<const FilterConfig*>(route_config.get());
   EXPECT_EQ(config->limit(), 10);
   EXPECT_EQ(config->fillInterval().count(), 50);
+  // default trailers
+  EXPECT_EQ(const_cast<FilterConfig*>(config)->requestDelayTrailer(),
+            Http::LowerCaseString("bandwidth-request-delay-ms"));
+  EXPECT_EQ(const_cast<FilterConfig*>(config)->responseDelayTrailer(),
+            Http::LowerCaseString("bandwidth-response-delay-ms"));
 }
 
 TEST(Factory, PerRouteConfigNoLimits) {