From d657bc6074a3f969129b87c75fc2fa964fb197a6 Mon Sep 17 00:00:00 2001 From: gayang Date: Mon, 27 Sep 2021 01:22:26 +0000 Subject: [PATCH 01/18] add response trailers for bandwidth limit Signed-off-by: gayang --- .../v3alpha/bandwidth_limit.proto | 12 ++- .../http_filters/bandwidth_limit_filter.rst | 10 ++- docs/root/version_history/current.rst | 1 + .../http/bandwidth_limit/bandwidth_limit.cc | 62 +++++++++++-- .../http/bandwidth_limit/bandwidth_limit.h | 16 +++- .../http/common/stream_rate_limiter.cc | 14 ++- .../filters/http/common/stream_rate_limiter.h | 10 +-- .../filters/http/fault/fault_filter.cc | 2 +- .../http/bandwidth_limit/config_test.cc | 10 +++ .../http/bandwidth_limit/filter_test.cc | 88 +++++++++++++------ .../http/common/stream_rate_limiter_test.cc | 4 +- 11 files changed, 176 insertions(+), 53 deletions(-) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto index 4cd5f8268b704..c47878863195f 100644 --- a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto @@ -20,7 +20,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Bandwidth limit :ref:`configuration overview `. // [#extension: envoy.filters.http.bandwidth_limit] -// [#next-free-field: 6] +// [#next-free-field: 7] message BandwidthLimit { // Defines the mode for the bandwidth limit filter. // Values represent bitmask. @@ -67,4 +67,14 @@ message BandwidthLimit { // Runtime flag that controls whether the filter is enabled or not. If not specified, defaults // to enabled. config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; + + // Optional the prefix for the response trailers of bandwidth decode/encode delays. + // If not set, use the default value "bandwidth-request-delay-ms" and "bandwidth-response-delay-ms". + // If set, the trailer name will be set as: + // request: response_trailer_prefix + "-bandwidth-request-delay-ms" + // Delay time it took for the request stream transfer. + // response: response_trailer_prefix + "-bandwidth-response-delay-ms" + // Delay time it took for the response stream transfer. + // If EnableMode is Disabled or Decode or delay time = 0, the trailer will not be set. + string response_trailer_prefix = 6; } diff --git a/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst b/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst index 4576e9d3ac36a..5c032496b723a 100644 --- a/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst +++ b/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst @@ -42,14 +42,16 @@ The HTTP bandwidth limit filter outputs statistics in the ``.http_b :widths: 1, 1, 2 request_enabled, Counter, Total number of request streams for which the bandwidth limiter was consulted + request_enforced, Counter, Total number of request streams for which the bandwidth limiter was enforced request_pending, GAUGE, Number of request streams which are currently pending transfer in bandwidth limiter - request_incoming_size, GAUGE, Size in bytes of incoming request data to bandwidth limiter - request_allowed_size, GAUGE, Size in bytes of outgoing request data from bandwidth limiter + request_incoming_size, Counter, Size in bytes of incoming request data to bandwidth limiter + request_allowed_size, Counter, Size in bytes of outgoing request data from bandwidth limiter request_transfer_duration, HISTOGRAM, Total time (including added delay) it took for the request stream transfer response_enabled, Counter, Total number of response streams for which the bandwidth limiter was consulted + response_enforced, Counter, Total number of response streams for which the bandwidth limiter was enforced response_pending, GAUGE, Number of response streams which are currently pending transfer in bandwidth limiter - response_incoming_size, GAUGE, Size in bytes of incoming response data to bandwidth limiter - response_allowed_size, GAUGE, Size in bytes of outgoing response data from bandwidth limiter + response_incoming_size, Counter, Size in bytes of incoming response data to bandwidth limiter + response_allowed_size, Counter, Size in bytes of outgoing response data from bandwidth limiter response_transfer_duration, HISTOGRAM, Total time (including added delay) it took for the response stream transfer .. _config_http_filters_bandwidth_limit_runtime: diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 8a3a2d6a1f08c..28b181d36db96 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -65,6 +65,7 @@ Minor Behavior Changes to false. As part of this change, the use of reuse_port for TCP listeners on both macOS and Windows has been disabled due to suboptimal behavior. See the field documentation for more information. +* bandwidth_limit: added response trailers when request or response delay are enforced. * listener: destroy per network filter chain stats when a network filter chain is removed during the listener in-place update. * quic: enables IETF connection migration. This feature requires a stable UDP packet routine in the L4 load balancer with the same first-4-bytes in connection id. It can be turned off by setting runtime guard ``envoy.reloadable_features.FLAGS_quic_reloadable_flag_quic_connection_migration_use_new_cid_v2`` to false. diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc index 6d831d6de8903..89c34b5b31a51 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc @@ -16,6 +16,13 @@ namespace Extensions { namespace HttpFilters { namespace BandwidthLimitFilter { +namespace { +const Http::LowerCaseString DefaultRequestDelayTrailer = + Http::LowerCaseString("bandwidth-request-delay-ms"); +const Http::LowerCaseString DefaultResponseDelayTrailer = + Http::LowerCaseString("bandwidth-response-delay-ms"); +} // namespace + FilterConfig::FilterConfig(const BandwidthLimit& config, Stats::Scope& scope, Runtime::Loader& runtime, TimeSource& time_source, bool per_route) : runtime_(runtime), time_source_(time_source), enable_mode_(config.enable_mode()), @@ -23,7 +30,15 @@ FilterConfig::FilterConfig(const BandwidthLimit& config, Stats::Scope& scope, fill_interval_(std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT( config, fill_interval, StreamRateLimiter::DefaultFillInterval.count()))), enabled_(config.runtime_enabled(), runtime), - stats_(generateStats(config.stat_prefix(), scope)) { + stats_(generateStats(config.stat_prefix(), scope)), + request_delay_trailer_(config.response_trailer_prefix().empty() + ? DefaultRequestDelayTrailer + : Http::LowerCaseString(config.response_trailer_prefix() + "-" + + DefaultRequestDelayTrailer.get())), + response_delay_trailer_(config.response_trailer_prefix().empty() + ? DefaultResponseDelayTrailer + : Http::LowerCaseString(config.response_trailer_prefix() + "-" + + DefaultResponseDelayTrailer.get())) { if (per_route && !config.has_limit_kbps()) { throw EnvoyException("bandwidthlimitfilter: limit must be set for per route filter config"); } @@ -64,7 +79,12 @@ Http::FilterHeadersStatus BandwidthLimiter::decodeHeaders(Http::RequestHeaderMap updateStatsOnDecodeFinish(); decoder_callbacks_->continueDecoding(); }, - [config](uint64_t len) { config.stats().request_allowed_size_.set(len); }, + [&config](uint64_t len, bool limit_enforced) { + config.stats().request_allowed_size_.add(len); + if (limit_enforced) { + config.stats().request_enforced_.inc(); + } + }, const_cast(&config)->timeSource(), decoder_callbacks_->dispatcher(), decoder_callbacks_->scope(), config.tokenBucket(), config.fillInterval()); } @@ -82,7 +102,7 @@ Http::FilterDataStatus BandwidthLimiter::decodeData(Buffer::Instance& data, bool const_cast(&config)->timeSource()); config.stats().request_pending_.inc(); } - config.stats().request_incoming_size_.set(data.length()); + config.stats().request_incoming_size_.add(data.length()); request_limiter_->writeData(data, end_stream); return Http::FilterDataStatus::StopIterationNoBuffer; @@ -123,7 +143,12 @@ Http::FilterHeadersStatus BandwidthLimiter::encodeHeaders(Http::ResponseHeaderMa updateStatsOnEncodeFinish(); encoder_callbacks_->continueEncoding(); }, - [config](uint64_t len) { config.stats().response_allowed_size_.set(len); }, + [&config](uint64_t len, bool limit_enforced) { + config.stats().response_allowed_size_.add(len); + if (limit_enforced) { + config.stats().response_enforced_.inc(); + } + }, const_cast(&config)->timeSource(), encoder_callbacks_->dispatcher(), encoder_callbacks_->scope(), config.tokenBucket(), config.fillInterval()); } @@ -135,23 +160,34 @@ Http::FilterDataStatus BandwidthLimiter::encodeData(Buffer::Instance& data, bool if (response_limiter_ != nullptr) { const auto& config = getConfig(); + // Adds encoded trailers. May only be called in encodeData when end_stream is set to true. + // If upstream has trailers, addEncodedTrailers won't be called + bool trailer_added = false; + if (end_stream) { + trailers = &encoder_callbacks_->addEncodedTrailers(); + trailer_added = true; + } + if (!response_latency_) { response_latency_ = std::make_unique( config.stats().response_transfer_duration_, const_cast(&config)->timeSource()); config.stats().response_pending_.inc(); } - config.stats().response_incoming_size_.set(data.length()); + config.stats().response_incoming_size_.add(data.length()); - response_limiter_->writeData(data, end_stream); + response_limiter_->writeData(data, end_stream, trailer_added); return Http::FilterDataStatus::StopIterationNoBuffer; } ENVOY_LOG(debug, "BandwidthLimiter : response_limiter not set"); return Http::FilterDataStatus::Continue; } -Http::FilterTrailersStatus BandwidthLimiter::encodeTrailers(Http::ResponseTrailerMap&) { +Http::FilterTrailersStatus +BandwidthLimiter::encodeTrailers(Http::ResponseTrailerMap& responseTrailers) { if (response_limiter_ != nullptr) { + trailers = &responseTrailers; + if (response_limiter_->onTrailers()) { return Http::FilterTrailersStatus::StopIteration; } else { @@ -164,6 +200,7 @@ Http::FilterTrailersStatus BandwidthLimiter::encodeTrailers(Http::ResponseTraile void BandwidthLimiter::updateStatsOnDecodeFinish() { if (request_latency_) { + request_duration_ = request_latency_.get()->elapsed().count(); request_latency_->complete(); request_latency_.reset(); getConfig().stats().request_pending_.dec(); @@ -172,9 +209,18 @@ void BandwidthLimiter::updateStatsOnDecodeFinish() { void BandwidthLimiter::updateStatsOnEncodeFinish() { if (response_latency_) { + const auto& config = getConfig(); + + auto response_duration = response_latency_.get()->elapsed().count(); + if (trailers != nullptr && request_duration_ > 0) { + trailers->setCopy(config.request_delay_trailer(), std::to_string(request_duration_)); + } + if (trailers != nullptr && response_duration > 0) { + trailers->setCopy(config.response_delay_trailer(), std::to_string(response_duration)); + } response_latency_->complete(); response_latency_.reset(); - getConfig().stats().response_pending_.dec(); + config.stats().response_pending_.dec(); } } diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h index f5bac46426425..40ba10e45ef72 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h @@ -32,12 +32,14 @@ namespace BandwidthLimitFilter { #define ALL_BANDWIDTH_LIMIT_STATS(COUNTER, GAUGE, HISTOGRAM) \ COUNTER(request_enabled) \ COUNTER(response_enabled) \ + COUNTER(request_enforced) \ + COUNTER(response_enforced) \ GAUGE(request_pending, Accumulate) \ GAUGE(response_pending, Accumulate) \ - GAUGE(request_incoming_size, Accumulate) \ - GAUGE(response_incoming_size, Accumulate) \ - GAUGE(request_allowed_size, Accumulate) \ - GAUGE(response_allowed_size, Accumulate) \ + COUNTER(request_incoming_size) \ + COUNTER(response_incoming_size) \ + COUNTER(request_allowed_size) \ + COUNTER(response_allowed_size) \ HISTOGRAM(request_transfer_duration, Milliseconds) \ HISTOGRAM(response_transfer_duration, Milliseconds) @@ -71,6 +73,8 @@ class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig { EnableMode enableMode() const { return enable_mode_; }; const std::shared_ptr tokenBucket() const { return token_bucket_; } std::chrono::milliseconds fillInterval() const { return fill_interval_; } + const Http::LowerCaseString& request_delay_trailer() const { return request_delay_trailer_; } + const Http::LowerCaseString& response_delay_trailer() const { return response_delay_trailer_; } private: friend class FilterTest; @@ -86,6 +90,8 @@ class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig { mutable BandwidthLimitStats stats_; // Filter chain's shared token bucket std::shared_ptr token_bucket_; + const Http::LowerCaseString request_delay_trailer_; + const Http::LowerCaseString response_delay_trailer_; }; using FilterConfigSharedPtr = std::shared_ptr; @@ -142,6 +148,8 @@ class BandwidthLimiter : public Http::StreamFilter, Logger::Loggable response_limiter_; Stats::TimespanPtr request_latency_; Stats::TimespanPtr response_latency_; + uint64_t request_duration_ = 0; + Http::ResponseTrailerMap* trailers; }; } // namespace BandwidthLimitFilter diff --git a/source/extensions/filters/http/common/stream_rate_limiter.cc b/source/extensions/filters/http/common/stream_rate_limiter.cc index 6763adbeb2414..a4e215a5f1d03 100644 --- a/source/extensions/filters/http/common/stream_rate_limiter.cc +++ b/source/extensions/filters/http/common/stream_rate_limiter.cc @@ -17,7 +17,7 @@ StreamRateLimiter::StreamRateLimiter( uint64_t max_kbps, uint64_t max_buffered_data, std::function pause_data_cb, std::function resume_data_cb, std::function write_data_cb, std::function continue_cb, - std::function write_stats_cb, TimeSource& time_source, + std::function write_stats_cb, TimeSource& time_source, Event::Dispatcher& dispatcher, const ScopeTrackedObject& scope, std::shared_ptr token_bucket, std::chrono::milliseconds fill_interval) : fill_interval_(std::move(fill_interval)), write_data_cb_(write_data_cb), @@ -63,7 +63,7 @@ void StreamRateLimiter::onTokenTimer() { // Move the data to write into the output buffer with as little copying as possible. // NOTE: This might be moving zero bytes, but that should work fine. data_to_write.move(buffer_, bytes_to_write); - write_stats_cb_(bytes_to_write); + write_stats_cb_(bytes_to_write, buffer_.length() > 0); // If the buffer still contains data in it, we couldn't get enough tokens, so schedule the next // token available time. @@ -88,10 +88,18 @@ void StreamRateLimiter::onTokenTimer() { } } -void StreamRateLimiter::writeData(Buffer::Instance& incoming_buffer, bool end_stream) { +void StreamRateLimiter::writeData(Buffer::Instance& incoming_buffer, bool end_stream, + bool trailer_added) { auto len = incoming_buffer.length(); buffer_.move(incoming_buffer); saw_end_stream_ = end_stream; + // If trailer_added is true, set saw_trailers_ to true to continue encode trailers, added + // after buffer_.move to ensure buffer has data and won't invoke continue_cb_ before + // processing the data in last data frame. + if (trailer_added) { + saw_trailers_ = true; + } + ENVOY_LOG(debug, "StreamRateLimiter : got new {} bytes of data. token " "timer {} scheduled.", diff --git a/source/extensions/filters/http/common/stream_rate_limiter.h b/source/extensions/filters/http/common/stream_rate_limiter.h index b8aed9ac8a72a..42b6c21015f91 100644 --- a/source/extensions/filters/http/common/stream_rate_limiter.h +++ b/source/extensions/filters/http/common/stream_rate_limiter.h @@ -49,9 +49,9 @@ class StreamRateLimiter : Logger::Loggable { StreamRateLimiter(uint64_t max_kbps, uint64_t max_buffered_data, std::function pause_data_cb, std::function resume_data_cb, std::function write_data_cb, - std::function continue_cb, std::function write_stats_cb, - TimeSource& time_source, Event::Dispatcher& dispatcher, - const ScopeTrackedObject& scope, + std::function continue_cb, + std::function write_stats_cb, TimeSource& time_source, + Event::Dispatcher& dispatcher, const ScopeTrackedObject& scope, std::shared_ptr token_bucket = nullptr, std::chrono::milliseconds fill_interval = DefaultFillInterval); @@ -59,7 +59,7 @@ class StreamRateLimiter : Logger::Loggable { * Called by the stream to write data. All data writes happen asynchronously, the stream should * be stopped after this call (all data will be drained from incoming_buffer). */ - void writeData(Buffer::Instance& incoming_buffer, bool end_stream); + void writeData(Buffer::Instance& incoming_buffer, bool end_stream, bool trailer_added = false); /** * Called if the stream receives trailers. @@ -83,7 +83,7 @@ class StreamRateLimiter : Logger::Loggable { const std::chrono::milliseconds fill_interval_; const std::function write_data_cb_; const std::function continue_cb_; - const std::function write_stats_cb_; + const std::function write_stats_cb_; const ScopeTrackedObject& scope_; std::shared_ptr token_bucket_; Event::TimerPtr token_timer_; diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index 7499e47dd5b8c..809b396b2b0fc 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -212,7 +212,7 @@ void FaultFilter::maybeSetupResponseRateLimit(const Http::RequestHeaderMap& requ encoder_callbacks_->injectEncodedDataToFilterChain(data, end_stream); }, [this] { encoder_callbacks_->continueEncoding(); }, - [](uint64_t) { + [](uint64_t, bool) { // write stats callback. }, config_->timeSource(), decoder_callbacks_->dispatcher(), decoder_callbacks_->scope()); diff --git a/test/extensions/filters/http/bandwidth_limit/config_test.cc b/test/extensions/filters/http/bandwidth_limit/config_test.cc index b98ea7ca4d67f..e3a16aa07a9b8 100644 --- a/test/extensions/filters/http/bandwidth_limit/config_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/config_test.cc @@ -38,6 +38,7 @@ TEST(Factory, RouteSpecificFilterConfig) { enable_mode: REQUEST_AND_RESPONSE limit_kbps: 10 fill_interval: 0.1s + response_trailer_prefix: test )"; BandwidthLimitFilterConfig factory; @@ -54,6 +55,10 @@ TEST(Factory, RouteSpecificFilterConfig) { EXPECT_EQ(config->fillInterval().count(), 100); EXPECT_EQ(config->enableMode(), EnableMode::BandwidthLimit_EnableMode_REQUEST_AND_RESPONSE); EXPECT_FALSE(config->tokenBucket() == nullptr); + EXPECT_EQ(const_cast(config)->request_delay_trailer(), + Http::LowerCaseString("test-bandwidth-request-delay-ms")); + EXPECT_EQ(const_cast(config)->response_delay_trailer(), + Http::LowerCaseString("test-bandwidth-response-delay-ms")); } TEST(Factory, RouteSpecificFilterConfigDisabledByDefault) { @@ -97,6 +102,11 @@ TEST(Factory, RouteSpecificFilterConfigDefaultFillInterval) { const auto* config = dynamic_cast(route_config.get()); EXPECT_EQ(config->limit(), 10); EXPECT_EQ(config->fillInterval().count(), 50); + // default trailers + EXPECT_EQ(const_cast(config)->request_delay_trailer(), + Http::LowerCaseString("bandwidth-request-delay-ms")); + EXPECT_EQ(const_cast(config)->response_delay_trailer(), + Http::LowerCaseString("bandwidth-response-delay-ms")); } TEST(Factory, PerRouteConfigNoLimits) { diff --git a/test/extensions/filters/http/bandwidth_limit/filter_test.cc b/test/extensions/filters/http/bandwidth_limit/filter_test.cc index daffe9076ea34..3977f0fb5ad3d 100644 --- a/test/extensions/filters/http/bandwidth_limit/filter_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/filter_test.cc @@ -11,6 +11,7 @@ using testing::_; using testing::AnyNumber; using testing::NiceMock; using testing::Return; +using testing::ReturnRef; namespace Envoy { namespace Extensions { @@ -57,6 +58,7 @@ class FilterTest : public testing::Test { Http::TestResponseTrailerMapImpl response_trailers_; Buffer::OwnedImpl data_; Event::SimulatedTimeSystem time_system_; + Http::TestResponseTrailerMapImpl trailers_; }; TEST_F(FilterTest, Disabled) { @@ -75,11 +77,14 @@ TEST_F(FilterTest, Disabled) { EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers_)); EXPECT_EQ(0U, findCounter("test.http_bandwidth_limit.request_enabled")); + EXPECT_EQ(0U, findCounter("test.http_bandwidth_limit.request_enforced")); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers_, false)); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(data_, false)); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); EXPECT_EQ(0U, findCounter("test.http_bandwidth_limit.response_enabled")); + EXPECT_EQ(false, response_trailers_.has("bandwidth-request-delay-ms")); + EXPECT_EQ(false, response_trailers_.has("bandwidth-response-delay-ms")); } TEST_F(FilterTest, LimitOnDecode) { @@ -90,6 +95,7 @@ TEST_F(FilterTest, LimitOnDecode) { runtime_key: foo_key enable_mode: REQUEST limit_kbps: 1 + response_trailer_prefix: test )"; setup(fmt::format(config_yaml, "1")); @@ -107,11 +113,12 @@ TEST_F(FilterTest, LimitOnDecode) { EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data1, false)); EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); - EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(5, findCounter("test.http_bandwidth_limit.request_incoming_size")); EXPECT_CALL(decoder_filter_callbacks_, injectDecodedDataToFilterChain(BufferStringEqual("hello"), false)); token_timer->invokeCallback(); - EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(0, findCounter("test.http_bandwidth_limit.request_enforced")); + EXPECT_EQ(5, findCounter("test.http_bandwidth_limit.request_allowed_size")); // Advance time by 1s which should refill all tokens. time_system_.advanceTimeWait(std::chrono::seconds(1)); @@ -122,15 +129,16 @@ TEST_F(FilterTest, LimitOnDecode) { Buffer::OwnedImpl data2(std::string(1126, 'a')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data2, false)); EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); - EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.request_incoming_size")); EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(50), _)); EXPECT_CALL(decoder_filter_callbacks_, onDecoderFilterBelowWriteBufferLowWatermark()); EXPECT_CALL(decoder_filter_callbacks_, injectDecodedDataToFilterChain(BufferStringEqual(std::string(1024, 'a')), false)); token_timer->invokeCallback(); - EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.request_allowed_size")); - EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(1, findCounter("test.http_bandwidth_limit.request_enforced")); + EXPECT_EQ(1029, findCounter("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.request_incoming_size")); // Fire timer, also advance time. time_system_.advanceTimeWait(std::chrono::milliseconds(50)); @@ -138,14 +146,15 @@ TEST_F(FilterTest, LimitOnDecode) { EXPECT_CALL(decoder_filter_callbacks_, injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); token_timer->invokeCallback(); - EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_allowed_size")); - EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(2, findCounter("test.http_bandwidth_limit.request_enforced")); + EXPECT_EQ(1080, findCounter("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.request_incoming_size")); // Get new data with current data buffered, not end_stream. Buffer::OwnedImpl data3(std::string(51, 'b')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data3, false)); EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); - EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(1182, findCounter("test.http_bandwidth_limit.request_incoming_size")); // Fire timer, also advance time. time_system_.advanceTimeWait(std::chrono::milliseconds(50)); @@ -153,7 +162,8 @@ TEST_F(FilterTest, LimitOnDecode) { EXPECT_CALL(decoder_filter_callbacks_, injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); token_timer->invokeCallback(); - EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.request_enforced")); + EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.request_allowed_size")); // Fire timer, also advance time. No timer enable because there is nothing // buffered. @@ -161,7 +171,8 @@ TEST_F(FilterTest, LimitOnDecode) { EXPECT_CALL(decoder_filter_callbacks_, injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'b')), false)); token_timer->invokeCallback(); - EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.request_enforced")); + EXPECT_EQ(1182, findCounter("test.http_bandwidth_limit.request_allowed_size")); // Advance time by 1s for a full refill. time_system_.advanceTimeWait(std::chrono::seconds(1)); @@ -171,12 +182,15 @@ TEST_F(FilterTest, LimitOnDecode) { EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); Buffer::OwnedImpl data4(std::string(1024, 'c')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data4, true)); - EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(2206, findCounter("test.http_bandwidth_limit.request_incoming_size")); EXPECT_CALL(decoder_filter_callbacks_, injectDecodedDataToFilterChain(BufferStringEqual(std::string(1024, 'c')), true)); token_timer->invokeCallback(); - EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.request_enforced")); + EXPECT_EQ(2206, findCounter("test.http_bandwidth_limit.request_allowed_size")); EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.request_pending")); + EXPECT_EQ(false, response_trailers_.has("test-bandwidth-request-delay-ms")); + EXPECT_EQ(false, response_trailers_.has("test-bandwidth-response-delay-ms")); filter_->onDestroy(); } @@ -189,10 +203,12 @@ TEST_F(FilterTest, LimitOnEncode) { runtime_key: foo_key enable_mode: RESPONSE limit_kbps: 1 + response_trailer_prefix: test )"; setup(fmt::format(config_yaml, "1")); ON_CALL(encoder_filter_callbacks_, encoderBufferLimit()).WillByDefault(Return(1100)); + ON_CALL(encoder_filter_callbacks_, addEncodedTrailers()).WillByDefault(ReturnRef(trailers_)); Event::MockTimer* token_timer = new NiceMock(&encoder_filter_callbacks_.dispatcher_); @@ -211,11 +227,12 @@ TEST_F(FilterTest, LimitOnEncode) { EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data1, false)); EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); - EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_EQ(5, findCounter("test.http_bandwidth_limit.response_incoming_size")); EXPECT_CALL(encoder_filter_callbacks_, injectEncodedDataToFilterChain(BufferStringEqual("hello"), false)); token_timer->invokeCallback(); - EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(0, findCounter("test.http_bandwidth_limit.response_enforced")); + EXPECT_EQ(5, findCounter("test.http_bandwidth_limit.response_allowed_size")); // Advance time by 1s which should refill all tokens. time_system_.advanceTimeWait(std::chrono::seconds(1)); @@ -225,7 +242,7 @@ TEST_F(FilterTest, LimitOnEncode) { EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); Buffer::OwnedImpl data2(std::string(1126, 'a')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data2, false)); - EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.response_incoming_size")); EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(50), _)); EXPECT_CALL(encoder_filter_callbacks_, onEncoderFilterBelowWriteBufferLowWatermark()); @@ -233,8 +250,9 @@ TEST_F(FilterTest, LimitOnEncode) { injectEncodedDataToFilterChain(BufferStringEqual(std::string(1024, 'a')), false)); token_timer->invokeCallback(); EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); - EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.response_incoming_size")); - EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(1, findCounter("test.http_bandwidth_limit.response_enforced")); + EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_EQ(1029, findCounter("test.http_bandwidth_limit.response_allowed_size")); // Fire timer, also advance time. time_system_.advanceTimeWait(std::chrono::milliseconds(50)); @@ -242,7 +260,8 @@ TEST_F(FilterTest, LimitOnEncode) { EXPECT_CALL(encoder_filter_callbacks_, injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); token_timer->invokeCallback(); - EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(2, findCounter("test.http_bandwidth_limit.response_enforced")); + EXPECT_EQ(1080, findCounter("test.http_bandwidth_limit.response_allowed_size")); // Get new data with current data buffered, not end_stream. Buffer::OwnedImpl data3(std::string(51, 'b')); @@ -254,7 +273,8 @@ TEST_F(FilterTest, LimitOnEncode) { EXPECT_CALL(encoder_filter_callbacks_, injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); token_timer->invokeCallback(); - EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.response_enforced")); + EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.response_allowed_size")); // Fire timer, also advance time. No time enable because there is nothing // buffered. @@ -262,7 +282,8 @@ TEST_F(FilterTest, LimitOnEncode) { EXPECT_CALL(encoder_filter_callbacks_, injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'b')), false)); token_timer->invokeCallback(); - EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.response_enforced")); + EXPECT_EQ(1182, findCounter("test.http_bandwidth_limit.response_allowed_size")); // Advance time by 1s for a full refill. time_system_.advanceTimeWait(std::chrono::seconds(1)); @@ -272,12 +293,16 @@ TEST_F(FilterTest, LimitOnEncode) { EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); Buffer::OwnedImpl data4(std::string(1024, 'c')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data4, true)); - EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_EQ(2206, findCounter("test.http_bandwidth_limit.response_incoming_size")); EXPECT_CALL(encoder_filter_callbacks_, - injectEncodedDataToFilterChain(BufferStringEqual(std::string(1024, 'c')), true)); + injectEncodedDataToFilterChain(BufferStringEqual(std::string(1024, 'c')), false)); token_timer->invokeCallback(); EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.response_pending")); - EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.response_enforced")); + EXPECT_EQ(2206, findCounter("test.http_bandwidth_limit.response_allowed_size")); + + EXPECT_EQ(false, response_trailers_.has("test-bandwidth-request-delay-ms")); + EXPECT_EQ("2150", trailers_.get_("test-bandwidth-response-delay-ms")); filter_->onDestroy(); } @@ -290,11 +315,13 @@ TEST_F(FilterTest, LimitOnDecodeAndEncode) { runtime_key: foo_key enable_mode: REQUEST_AND_RESPONSE limit_kbps: 1 + response_trailer_prefix: test )"; setup(fmt::format(config_yaml, "1")); ON_CALL(decoder_filter_callbacks_, decoderBufferLimit()).WillByDefault(Return(1050)); ON_CALL(encoder_filter_callbacks_, encoderBufferLimit()).WillByDefault(Return(1100)); + ON_CALL(encoder_filter_callbacks_, addEncodedTrailers()).WillByDefault(ReturnRef(trailers_)); Event::MockTimer* request_timer = new NiceMock(&decoder_filter_callbacks_.dispatcher_); Event::MockTimer* response_timer = @@ -403,9 +430,13 @@ TEST_F(FilterTest, LimitOnDecodeAndEncode) { EXPECT_CALL(decoder_filter_callbacks_, injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'd')), true)); EXPECT_CALL(encoder_filter_callbacks_, - injectEncodedDataToFilterChain(BufferStringEqual(std::string(960, 'e')), true)); - response_timer->invokeCallback(); + injectEncodedDataToFilterChain(BufferStringEqual(std::string(960, 'e')), false)); + EXPECT_CALL(encoder_filter_callbacks_, continueEncoding()); + request_timer->invokeCallback(); + response_timer->invokeCallback(); + EXPECT_EQ("2200", trailers_.get_("test-bandwidth-request-delay-ms")); + EXPECT_EQ("2200", trailers_.get_("test-bandwidth-response-delay-ms")); filter_->onDestroy(); } @@ -418,6 +449,7 @@ TEST_F(FilterTest, WithTrailers) { runtime_key: foo_key enable_mode: REQUEST_AND_RESPONSE limit_kbps: 1 + response_trailer_prefix: test )"; setup(fmt::format(config_yaml, "1")); @@ -479,6 +511,9 @@ TEST_F(FilterTest, WithTrailers) { injectEncodedDataToFilterChain(BufferStringEqual(std::string(5, 'e')), false)); response_timer->invokeCallback(); EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.response_pending")); + + EXPECT_EQ("50", response_trailers_.get_("test-bandwidth-request-delay-ms")); + EXPECT_EQ("150", response_trailers_.get_("test-bandwidth-response-delay-ms")); } TEST_F(FilterTest, WithTrailersNoEndStream) { @@ -550,6 +585,9 @@ TEST_F(FilterTest, WithTrailersNoEndStream) { EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers_)); EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.response_pending")); + + EXPECT_EQ("50", response_trailers_.get_("bandwidth-request-delay-ms")); + EXPECT_EQ("150", response_trailers_.get_("bandwidth-response-delay-ms")); } } // namespace BandwidthLimitFilter diff --git a/test/extensions/filters/http/common/stream_rate_limiter_test.cc b/test/extensions/filters/http/common/stream_rate_limiter_test.cc index 97f18f5957ca1..7fc0aeca1c66a 100644 --- a/test/extensions/filters/http/common/stream_rate_limiter_test.cc +++ b/test/extensions/filters/http/common/stream_rate_limiter_test.cc @@ -40,7 +40,7 @@ class StreamRateLimiterTest : public testing::Test { decoder_callbacks_.injectDecodedDataToFilterChain(data, end_stream); }, [this] { decoder_callbacks_.continueDecoding(); }, - [](uint64_t /*len*/) { + [](uint64_t /*len*/, bool) { // config->stats().decode_allowed_size_.set(len); }, time_system_, decoder_callbacks_.dispatcher_, decoder_callbacks_.scope(), token_bucket, @@ -59,7 +59,7 @@ class StreamRateLimiterTest : public testing::Test { decoder_callbacks_.injectDecodedDataToFilterChain(data, end_stream); }, [this] { decoder_callbacks_.continueDecoding(); }, - [](uint64_t /*len*/) { + [](uint64_t /*len*/, bool) { // config->stats().decode_allowed_size_.set(len); }, time_system_, decoder_callbacks_.dispatcher_, decoder_callbacks_.scope()); From f27b1c7c60d2ee96ae214ba9846104c18ea33875 Mon Sep 17 00:00:00 2001 From: gayang Date: Mon, 27 Sep 2021 02:20:40 +0000 Subject: [PATCH 02/18] fix format Signed-off-by: gayang --- .../filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto | 2 +- docs/root/version_history/current.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto index c47878863195f..6b42299251902 100644 --- a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto @@ -77,4 +77,4 @@ message BandwidthLimit { // Delay time it took for the response stream transfer. // If EnableMode is Disabled or Decode or delay time = 0, the trailer will not be set. string response_trailer_prefix = 6; -} +} \ No newline at end of file diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 28b181d36db96..6d2804fc1433f 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -32,6 +32,7 @@ Minor Behavior Changes ---------------------- *Changes that may cause incompatibilities for some users, but should not for most* +* bandwidth_limit: added response trailers when request or response delay are enforced. * client_ssl_auth filter: now sets additional termination details and **UAEX** response flag when the client certificate is not in the allowed-list. * config: configuration files ending in .yml now load as YAML. * config: configuration file extensions now ignore case when deciding the file type. E.g., .JSON file load as JSON. @@ -65,7 +66,6 @@ Minor Behavior Changes to false. As part of this change, the use of reuse_port for TCP listeners on both macOS and Windows has been disabled due to suboptimal behavior. See the field documentation for more information. -* bandwidth_limit: added response trailers when request or response delay are enforced. * listener: destroy per network filter chain stats when a network filter chain is removed during the listener in-place update. * quic: enables IETF connection migration. This feature requires a stable UDP packet routine in the L4 load balancer with the same first-4-bytes in connection id. It can be turned off by setting runtime guard ``envoy.reloadable_features.FLAGS_quic_reloadable_flag_quic_connection_migration_use_new_cid_v2`` to false. From 0e063bfada1e1086851c7a3971e246453fc0e442 Mon Sep 17 00:00:00 2001 From: gayang Date: Mon, 27 Sep 2021 02:22:13 +0000 Subject: [PATCH 03/18] fix format Signed-off-by: gayang --- .../filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto index 6b42299251902..c47878863195f 100644 --- a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto @@ -77,4 +77,4 @@ message BandwidthLimit { // Delay time it took for the response stream transfer. // If EnableMode is Disabled or Decode or delay time = 0, the trailer will not be set. string response_trailer_prefix = 6; -} \ No newline at end of file +} From fee39ec9066bc065dddaa0dcafb2e5782631607c Mon Sep 17 00:00:00 2001 From: gayang Date: Mon, 27 Sep 2021 13:18:06 +0000 Subject: [PATCH 04/18] fix doc format Signed-off-by: gayang --- .../http/bandwidth_limit/v3alpha/bandwidth_limit.proto | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto index c47878863195f..0d16632a04882 100644 --- a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto @@ -68,13 +68,11 @@ message BandwidthLimit { // to enabled. config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; - // Optional the prefix for the response trailers of bandwidth decode/encode delays. + // Optional the prefix for the response trailers of bandwidth decode/encode delays for stream transfer. // If not set, use the default value "bandwidth-request-delay-ms" and "bandwidth-response-delay-ms". // If set, the trailer name will be set as: - // request: response_trailer_prefix + "-bandwidth-request-delay-ms" - // Delay time it took for the request stream transfer. - // response: response_trailer_prefix + "-bandwidth-response-delay-ms" - // Delay time it took for the response stream transfer. - // If EnableMode is Disabled or Decode or delay time = 0, the trailer will not be set. + // request: response_trailer_prefix + "-bandwidth-request-delay-ms" + // response: response_trailer_prefix + "-bandwidth-response-delay-ms" + // If EnableMode is DISABLED or REQUEST or delay time = 0, the trailer will not be set. string response_trailer_prefix = 6; } From 13bd62486e49b42d1f605c0ad261607e42bfcba4 Mon Sep 17 00:00:00 2001 From: gayang Date: Tue, 28 Sep 2021 02:26:09 +0000 Subject: [PATCH 05/18] fix clang_tidy check Signed-off-by: gayang --- .../http/bandwidth_limit/bandwidth_limit.cc | 14 +++++++------- .../filters/http/bandwidth_limit/bandwidth_limit.h | 6 +++--- .../filters/http/bandwidth_limit/config_test.cc | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc index 89c34b5b31a51..1fff48472c89e 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc @@ -164,7 +164,7 @@ Http::FilterDataStatus BandwidthLimiter::encodeData(Buffer::Instance& data, bool // If upstream has trailers, addEncodedTrailers won't be called bool trailer_added = false; if (end_stream) { - trailers = &encoder_callbacks_->addEncodedTrailers(); + trailers_ = &encoder_callbacks_->addEncodedTrailers(); trailer_added = true; } @@ -184,9 +184,9 @@ Http::FilterDataStatus BandwidthLimiter::encodeData(Buffer::Instance& data, bool } Http::FilterTrailersStatus -BandwidthLimiter::encodeTrailers(Http::ResponseTrailerMap& responseTrailers) { +BandwidthLimiter::encodeTrailers(Http::ResponseTrailerMap& response_trailers) { if (response_limiter_ != nullptr) { - trailers = &responseTrailers; + trailers_ = &response_trailers; if (response_limiter_->onTrailers()) { return Http::FilterTrailersStatus::StopIteration; @@ -212,11 +212,11 @@ void BandwidthLimiter::updateStatsOnEncodeFinish() { const auto& config = getConfig(); auto response_duration = response_latency_.get()->elapsed().count(); - if (trailers != nullptr && request_duration_ > 0) { - trailers->setCopy(config.request_delay_trailer(), std::to_string(request_duration_)); + if (trailers_ != nullptr && request_duration_ > 0) { + trailers_->setCopy(config.requestDelayTrailer(), std::to_string(request_duration_)); } - if (trailers != nullptr && response_duration > 0) { - trailers->setCopy(config.response_delay_trailer(), std::to_string(response_duration)); + if (trailers_ != nullptr && response_duration > 0) { + trailers_->setCopy(config.responseDelayTrailer(), std::to_string(response_duration)); } response_latency_->complete(); response_latency_.reset(); diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h index 40ba10e45ef72..6aacd9e1e6aa7 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h @@ -73,8 +73,8 @@ class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig { EnableMode enableMode() const { return enable_mode_; }; const std::shared_ptr tokenBucket() const { return token_bucket_; } std::chrono::milliseconds fillInterval() const { return fill_interval_; } - const Http::LowerCaseString& request_delay_trailer() const { return request_delay_trailer_; } - const Http::LowerCaseString& response_delay_trailer() const { return response_delay_trailer_; } + const Http::LowerCaseString& requestDelayTrailer() const { return request_delay_trailer_; } + const Http::LowerCaseString& responseDelayTrailer() const { return response_delay_trailer_; } private: friend class FilterTest; @@ -149,7 +149,7 @@ class BandwidthLimiter : public Http::StreamFilter, Logger::LoggablefillInterval().count(), 100); EXPECT_EQ(config->enableMode(), EnableMode::BandwidthLimit_EnableMode_REQUEST_AND_RESPONSE); EXPECT_FALSE(config->tokenBucket() == nullptr); - EXPECT_EQ(const_cast(config)->request_delay_trailer(), + EXPECT_EQ(const_cast(config)->requestDelayTrailer(), Http::LowerCaseString("test-bandwidth-request-delay-ms")); - EXPECT_EQ(const_cast(config)->response_delay_trailer(), + EXPECT_EQ(const_cast(config)->responseDelayTrailer(), Http::LowerCaseString("test-bandwidth-response-delay-ms")); } @@ -103,9 +103,9 @@ TEST(Factory, RouteSpecificFilterConfigDefaultFillInterval) { EXPECT_EQ(config->limit(), 10); EXPECT_EQ(config->fillInterval().count(), 50); // default trailers - EXPECT_EQ(const_cast(config)->request_delay_trailer(), + EXPECT_EQ(const_cast(config)->requestDelayTrailer(), Http::LowerCaseString("bandwidth-request-delay-ms")); - EXPECT_EQ(const_cast(config)->response_delay_trailer(), + EXPECT_EQ(const_cast(config)->responseDelayTrailer(), Http::LowerCaseString("bandwidth-response-delay-ms")); } From 3eabf3384250268b029ba8912ffb5ebdda84f8ac Mon Sep 17 00:00:00 2001 From: gayang Date: Tue, 28 Sep 2021 05:19:49 +0000 Subject: [PATCH 06/18] update release notes Signed-off-by: gayang --- docs/root/version_history/current.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 6d2804fc1433f..8080ff5fec131 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -33,6 +33,7 @@ Minor Behavior Changes *Changes that may cause incompatibilities for some users, but should not for most* * bandwidth_limit: added response trailers when request or response delay are enforced. +* bandwidth_limit: added bandwidth limit stats request_enforced and response_enforced. * client_ssl_auth filter: now sets additional termination details and **UAEX** response flag when the client certificate is not in the allowed-list. * config: configuration files ending in .yml now load as YAML. * config: configuration file extensions now ignore case when deciding the file type. E.g., .JSON file load as JSON. From e7cc99e4b9c02866a489ad580bdd71815c773cd1 Mon Sep 17 00:00:00 2001 From: gayang Date: Wed, 29 Sep 2021 11:40:51 +0000 Subject: [PATCH 07/18] add attribute enable_response_trailer Signed-off-by: gayang --- .../v3alpha/bandwidth_limit.proto | 11 +++++++--- .../http/bandwidth_limit/bandwidth_limit.cc | 20 +++++++++++-------- .../http/bandwidth_limit/bandwidth_limit.h | 2 ++ .../http/bandwidth_limit/filter_test.cc | 8 +++++--- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto index 0d16632a04882..5e1bb29fffab7 100644 --- a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto @@ -20,7 +20,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Bandwidth limit :ref:`configuration overview `. // [#extension: envoy.filters.http.bandwidth_limit] -// [#next-free-field: 7] +// [#next-free-field: 8] message BandwidthLimit { // Defines the mode for the bandwidth limit filter. // Values represent bitmask. @@ -68,11 +68,16 @@ message BandwidthLimit { // to enabled. config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; + // Optional flag to enabled response trailers. + // Default is false. + bool enable_response_trailer = 6; + // Optional the prefix for the response trailers of bandwidth decode/encode delays for stream transfer. // If not set, use the default value "bandwidth-request-delay-ms" and "bandwidth-response-delay-ms". // If set, the trailer name will be set as: // request: response_trailer_prefix + "-bandwidth-request-delay-ms" // response: response_trailer_prefix + "-bandwidth-response-delay-ms" - // If EnableMode is DISABLED or REQUEST or delay time = 0, the trailer will not be set. - string response_trailer_prefix = 6; + // If enable_mode is DISABLED or REQUEST or delay time = 0, the trailer will not be set. + // If enable_response_trailer is false, the trailer will not be set. + string response_trailer_prefix = 7; } diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc index 1fff48472c89e..795ce93e43af9 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc @@ -38,7 +38,8 @@ FilterConfig::FilterConfig(const BandwidthLimit& config, Stats::Scope& scope, response_delay_trailer_(config.response_trailer_prefix().empty() ? DefaultResponseDelayTrailer : Http::LowerCaseString(config.response_trailer_prefix() + "-" + - DefaultResponseDelayTrailer.get())) { + DefaultResponseDelayTrailer.get())), + enable_response_trailer_(config.enable_response_trailer()) { if (per_route && !config.has_limit_kbps()) { throw EnvoyException("bandwidthlimitfilter: limit must be set for per route filter config"); } @@ -163,7 +164,7 @@ Http::FilterDataStatus BandwidthLimiter::encodeData(Buffer::Instance& data, bool // Adds encoded trailers. May only be called in encodeData when end_stream is set to true. // If upstream has trailers, addEncodedTrailers won't be called bool trailer_added = false; - if (end_stream) { + if (config.enableResponseTrailer() && end_stream) { trailers_ = &encoder_callbacks_->addEncodedTrailers(); trailer_added = true; } @@ -211,13 +212,16 @@ void BandwidthLimiter::updateStatsOnEncodeFinish() { if (response_latency_) { const auto& config = getConfig(); - auto response_duration = response_latency_.get()->elapsed().count(); - if (trailers_ != nullptr && request_duration_ > 0) { - trailers_->setCopy(config.requestDelayTrailer(), std::to_string(request_duration_)); - } - if (trailers_ != nullptr && response_duration > 0) { - trailers_->setCopy(config.responseDelayTrailer(), std::to_string(response_duration)); + if (config.enableResponseTrailer() && trailers_ != nullptr) { + auto response_duration = response_latency_.get()->elapsed().count(); + if (request_duration_ > 0) { + trailers_->setCopy(config.requestDelayTrailer(), std::to_string(request_duration_)); + } + if (response_duration > 0) { + trailers_->setCopy(config.responseDelayTrailer(), std::to_string(response_duration)); + } } + response_latency_->complete(); response_latency_.reset(); config.stats().response_pending_.dec(); diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h index 6aacd9e1e6aa7..835fe9b017085 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h @@ -75,6 +75,7 @@ class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig { std::chrono::milliseconds fillInterval() const { return fill_interval_; } const Http::LowerCaseString& requestDelayTrailer() const { return request_delay_trailer_; } const Http::LowerCaseString& responseDelayTrailer() const { return response_delay_trailer_; } + bool enableResponseTrailer() const {return enable_response_trailer_; } private: friend class FilterTest; @@ -92,6 +93,7 @@ class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig { std::shared_ptr token_bucket_; const Http::LowerCaseString request_delay_trailer_; const Http::LowerCaseString response_delay_trailer_; + const bool enable_response_trailer_; }; using FilterConfigSharedPtr = std::shared_ptr; diff --git a/test/extensions/filters/http/bandwidth_limit/filter_test.cc b/test/extensions/filters/http/bandwidth_limit/filter_test.cc index 3977f0fb5ad3d..e7734d8785f02 100644 --- a/test/extensions/filters/http/bandwidth_limit/filter_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/filter_test.cc @@ -203,6 +203,7 @@ TEST_F(FilterTest, LimitOnEncode) { runtime_key: foo_key enable_mode: RESPONSE limit_kbps: 1 + enable_response_trailer: true response_trailer_prefix: test )"; setup(fmt::format(config_yaml, "1")); @@ -315,6 +316,7 @@ TEST_F(FilterTest, LimitOnDecodeAndEncode) { runtime_key: foo_key enable_mode: REQUEST_AND_RESPONSE limit_kbps: 1 + enable_response_trailer: true response_trailer_prefix: test )"; setup(fmt::format(config_yaml, "1")); @@ -511,9 +513,8 @@ TEST_F(FilterTest, WithTrailers) { injectEncodedDataToFilterChain(BufferStringEqual(std::string(5, 'e')), false)); response_timer->invokeCallback(); EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.response_pending")); - - EXPECT_EQ("50", response_trailers_.get_("test-bandwidth-request-delay-ms")); - EXPECT_EQ("150", response_trailers_.get_("test-bandwidth-response-delay-ms")); + EXPECT_EQ(false, response_trailers_.has("test-bandwidth-request-delay-ms")); + EXPECT_EQ(false, response_trailers_.has("test-bandwidth-response-delay-ms")); } TEST_F(FilterTest, WithTrailersNoEndStream) { @@ -524,6 +525,7 @@ TEST_F(FilterTest, WithTrailersNoEndStream) { runtime_key: foo_key enable_mode: REQUEST_AND_RESPONSE limit_kbps: 1 + enable_response_trailer: true )"; setup(fmt::format(config_yaml, "1")); From a35a303f1a7b657e419ade05050b68c9e66e7d1f Mon Sep 17 00:00:00 2001 From: gayang Date: Wed, 29 Sep 2021 11:45:10 +0000 Subject: [PATCH 08/18] fix format Signed-off-by: gayang --- .../extensions/filters/http/bandwidth_limit/bandwidth_limit.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h index 835fe9b017085..f292ae023bb89 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h @@ -75,7 +75,7 @@ class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig { std::chrono::milliseconds fillInterval() const { return fill_interval_; } const Http::LowerCaseString& requestDelayTrailer() const { return request_delay_trailer_; } const Http::LowerCaseString& responseDelayTrailer() const { return response_delay_trailer_; } - bool enableResponseTrailer() const {return enable_response_trailer_; } + bool enableResponseTrailer() const { return enable_response_trailer_; } private: friend class FilterTest; From c1c73b39938aebab5c0e93a0fb2da17a1d5fff9f Mon Sep 17 00:00:00 2001 From: gayang Date: Thu, 30 Sep 2021 07:14:39 +0000 Subject: [PATCH 09/18] update api comments Signed-off-by: gayang --- .../v3alpha/bandwidth_limit.proto | 24 ++++++++++--------- .../http/bandwidth_limit/bandwidth_limit.cc | 6 ++--- .../http/bandwidth_limit/bandwidth_limit.h | 4 ++-- .../http/bandwidth_limit/filter_test.cc | 8 ++++--- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto index 5e1bb29fffab7..d1f69948b0447 100644 --- a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto @@ -68,16 +68,18 @@ message BandwidthLimit { // to enabled. config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; - // Optional flag to enabled response trailers. - // Default is false. - bool enable_response_trailer = 6; - - // Optional the prefix for the response trailers of bandwidth decode/encode delays for stream transfer. - // If not set, use the default value "bandwidth-request-delay-ms" and "bandwidth-response-delay-ms". - // If set, the trailer name will be set as: - // request: response_trailer_prefix + "-bandwidth-request-delay-ms" - // response: response_trailer_prefix + "-bandwidth-response-delay-ms" - // If enable_mode is DISABLED or REQUEST or delay time = 0, the trailer will not be set. - // If enable_response_trailer is false, the trailer will not be set. + // Optional Flag to enabled response trailers. + // + // .. note:: + // If set true, the response trailer "bandwidth-request-delay-ms" and "bandwidth-response-delay-ms" + // will be added. + // bandwidth-request-delay-ms: delay time in milliseconds it took for the request stream transfer. + // bandwidth-response-delay-ms: delay time in milliseconds it took for the response stream transfer. + // If enable_mode is DISABLED or REQUEST, the trailers will not be set. + // If the request/response delay time is 0, the trailers will not be set. + // + bool enable_response_trailers = 6; + + // Optional The prefix for the response trailers. string response_trailer_prefix = 7; } diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc index 795ce93e43af9..78363e2d99749 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc @@ -39,7 +39,7 @@ FilterConfig::FilterConfig(const BandwidthLimit& config, Stats::Scope& scope, ? DefaultResponseDelayTrailer : Http::LowerCaseString(config.response_trailer_prefix() + "-" + DefaultResponseDelayTrailer.get())), - enable_response_trailer_(config.enable_response_trailer()) { + enable_response_trailers_(config.enable_response_trailers()) { if (per_route && !config.has_limit_kbps()) { throw EnvoyException("bandwidthlimitfilter: limit must be set for per route filter config"); } @@ -164,7 +164,7 @@ Http::FilterDataStatus BandwidthLimiter::encodeData(Buffer::Instance& data, bool // Adds encoded trailers. May only be called in encodeData when end_stream is set to true. // If upstream has trailers, addEncodedTrailers won't be called bool trailer_added = false; - if (config.enableResponseTrailer() && end_stream) { + if (end_stream) { trailers_ = &encoder_callbacks_->addEncodedTrailers(); trailer_added = true; } @@ -212,7 +212,7 @@ void BandwidthLimiter::updateStatsOnEncodeFinish() { if (response_latency_) { const auto& config = getConfig(); - if (config.enableResponseTrailer() && trailers_ != nullptr) { + if (config.enableResponseTrailers() && trailers_ != nullptr) { auto response_duration = response_latency_.get()->elapsed().count(); if (request_duration_ > 0) { trailers_->setCopy(config.requestDelayTrailer(), std::to_string(request_duration_)); diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h index f292ae023bb89..bf8af664e6141 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h @@ -75,7 +75,7 @@ class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig { std::chrono::milliseconds fillInterval() const { return fill_interval_; } const Http::LowerCaseString& requestDelayTrailer() const { return request_delay_trailer_; } const Http::LowerCaseString& responseDelayTrailer() const { return response_delay_trailer_; } - bool enableResponseTrailer() const { return enable_response_trailer_; } + bool enableResponseTrailers() const { return enable_response_trailers_; } private: friend class FilterTest; @@ -93,7 +93,7 @@ class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig { std::shared_ptr token_bucket_; const Http::LowerCaseString request_delay_trailer_; const Http::LowerCaseString response_delay_trailer_; - const bool enable_response_trailer_; + const bool enable_response_trailers_; }; using FilterConfigSharedPtr = std::shared_ptr; diff --git a/test/extensions/filters/http/bandwidth_limit/filter_test.cc b/test/extensions/filters/http/bandwidth_limit/filter_test.cc index e7734d8785f02..e943d756d14c5 100644 --- a/test/extensions/filters/http/bandwidth_limit/filter_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/filter_test.cc @@ -70,6 +70,7 @@ TEST_F(FilterTest, Disabled) { enable_mode: DISABLED limit_kbps: 10 fill_interval: 1s + enable_response_trailers: true )"; setup(fmt::format(config_yaml, "1")); @@ -95,6 +96,7 @@ TEST_F(FilterTest, LimitOnDecode) { runtime_key: foo_key enable_mode: REQUEST limit_kbps: 1 + enable_response_trailers: true response_trailer_prefix: test )"; setup(fmt::format(config_yaml, "1")); @@ -203,7 +205,7 @@ TEST_F(FilterTest, LimitOnEncode) { runtime_key: foo_key enable_mode: RESPONSE limit_kbps: 1 - enable_response_trailer: true + enable_response_trailers: true response_trailer_prefix: test )"; setup(fmt::format(config_yaml, "1")); @@ -316,7 +318,7 @@ TEST_F(FilterTest, LimitOnDecodeAndEncode) { runtime_key: foo_key enable_mode: REQUEST_AND_RESPONSE limit_kbps: 1 - enable_response_trailer: true + enable_response_trailers: true response_trailer_prefix: test )"; setup(fmt::format(config_yaml, "1")); @@ -525,7 +527,7 @@ TEST_F(FilterTest, WithTrailersNoEndStream) { runtime_key: foo_key enable_mode: REQUEST_AND_RESPONSE limit_kbps: 1 - enable_response_trailer: true + enable_response_trailers: true )"; setup(fmt::format(config_yaml, "1")); From 77926bb454511474de3c1d0118f08e3f4d724b49 Mon Sep 17 00:00:00 2001 From: gayang Date: Thu, 30 Sep 2021 07:20:20 +0000 Subject: [PATCH 10/18] fix api comment typo Signed-off-by: gayang --- .../filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto index d1f69948b0447..150b8e88822da 100644 --- a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto @@ -68,7 +68,7 @@ message BandwidthLimit { // to enabled. config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; - // Optional Flag to enabled response trailers. + // Optional Flag to enable response trailers. // // .. note:: // If set true, the response trailer "bandwidth-request-delay-ms" and "bandwidth-response-delay-ms" From 938c35234d748dc319ce0e35ca8c56366bab93a6 Mon Sep 17 00:00:00 2001 From: gayang Date: Mon, 11 Oct 2021 03:40:44 +0000 Subject: [PATCH 11/18] update docs Signed-off-by: gayang --- .../v3alpha/bandwidth_limit.proto | 19 ++- docs/root/version_history/current.rst | 134 ++---------------- .../http/bandwidth_limit/bandwidth_limit.cc | 2 +- .../http/bandwidth_limit/bandwidth_limit.h | 6 +- .../http/bandwidth_limit/config_test.cc | 2 +- .../http/bandwidth_limit/filter_test.cc | 2 +- 6 files changed, 24 insertions(+), 141 deletions(-) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto index 150b8e88822da..c49aa52eec1a3 100644 --- a/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package envoy.extensions.filters.http.bandwidth_limit.v3alpha; +package envoy.extensions.filters.http.bandwidth_limit.v3; import "envoy/config/core/v3/base.proto"; @@ -10,10 +10,9 @@ import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; import "validate/validate.proto"; -option java_package = "io.envoyproxy.envoy.extensions.filters.http.bandwidth_limit.v3alpha"; +option java_package = "io.envoyproxy.envoy.extensions.filters.http.bandwidth_limit.v3"; option java_outer_classname = "BandwidthLimitProto"; option java_multiple_files = true; -option (udpa.annotations.file_status).work_in_progress = true; option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Bandwidth limit] @@ -68,15 +67,15 @@ message BandwidthLimit { // to enabled. config.core.v3.RuntimeFeatureFlag runtime_enabled = 5; - // Optional Flag to enable response trailers. + // Enable response trailers. // // .. note:: - // If set true, the response trailer "bandwidth-request-delay-ms" and "bandwidth-response-delay-ms" - // will be added. - // bandwidth-request-delay-ms: delay time in milliseconds it took for the request stream transfer. - // bandwidth-response-delay-ms: delay time in milliseconds it took for the response stream transfer. - // If enable_mode is DISABLED or REQUEST, the trailers will not be set. - // If the request/response delay time is 0, the trailers will not be set. + // 1. If set true, the response trailers *bandwidth-request-delay-ms* and *bandwidth-response-delay-ms* + // will be added, prefixed by *response_trailer_prefix*. + // 1. bandwidth-request-delay-ms: delay time in milliseconds it took for the request stream transfer. + // 1. bandwidth-response-delay-ms: delay time in milliseconds it took for the response stream transfer. + // 1. If :ref:`enable_mode ` is DISABLED or REQUEST, the trailers will not be set. + // 1. If the request/response delay time is 0, the trailers will not be set. // bool enable_response_trailers = 6; diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 8080ff5fec131..df9a4fbfbd59c 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -1,150 +1,34 @@ -1.20.0 (Pending) +1.21.0 (Pending) ================ Incompatible Behavior Changes ----------------------------- *Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* -* config: the ``--bootstrap-version`` CLI flag has been removed, Envoy has only been able to accept v3 - bootstrap configurations since 1.18.0. -* contrib: the :ref:`squash filter ` has been moved to - :ref:`contrib images `. -* contrib: the :ref:`kafka broker filter ` has been moved to - :ref:`contrib images `. -* contrib: the :ref:`RocketMQ proxy filter ` has been moved to - :ref:`contrib images `. -* contrib: the :ref:`Postgres proxy filter ` has been moved to - :ref:`contrib images `. -* contrib: the :ref:`MySQL proxy filter ` has been moved to - :ref:`contrib images `. -* dns_filter: :ref:`dns_filter ` - protobuf fields have been renumbered to restore compatibility with Envoy - 1.18, breaking compatibility with Envoy 1.19.0 and 1.19.1. The new field - numbering allows control planes supporting Envoy 1.18 to gracefully upgrade to - :ref:`dns_resolution_config `, - provided they skip over Envoy 1.19.0 and 1.19.1. - Control planes upgrading from Envoy 1.19.0 and 1.19.1 will need to - vendor the corresponding protobuf definitions to ensure that the - renumbered fields have the types expected by those releases. -* ext_authz: fixed skipping authentication when returning either a direct response or a redirect. This behavior can be temporarily reverted by setting the ``envoy.reloadable_features.http_ext_authz_do_not_skip_direct_response_and_redirect`` runtime guard to false. - Minor Behavior Changes ---------------------- *Changes that may cause incompatibilities for some users, but should not for most* -* bandwidth_limit: added response trailers when request or response delay are enforced. -* bandwidth_limit: added bandwidth limit stats request_enforced and response_enforced. -* client_ssl_auth filter: now sets additional termination details and **UAEX** response flag when the client certificate is not in the allowed-list. -* config: configuration files ending in .yml now load as YAML. -* config: configuration file extensions now ignore case when deciding the file type. E.g., .JSON file load as JSON. -* config: reduced log level for "Unable to establish new stream" xDS logs to debug. The log level - for "gRPC config stream closed" is now reduced to debug when the status is ``Ok`` or has been - retriable (``DeadlineExceeded``, ``ResourceExhausted``, or ``Unavailable``) for less than 30 - seconds. -* grpc: gRPC async client can be cached and shared across filter instances in the same thread, this feature is turned off by default, can be turned on by setting runtime guard ``envoy.reloadable_features.enable_grpc_async_client_cache`` to true. -* http: correct the use of the ``x-forwarded-proto`` header and the ``:scheme`` header. Where they differ - (which is rare) ``:scheme`` will now be used for serving redirect URIs and cached content. This behavior - can be reverted by setting runtime guard ``correct_scheme_and_xfp`` to false. -* http: reject requests with #fragment in the URI path. The fragment is not allowed to be part of the request - URI according to RFC3986 (3.5), RFC7230 (5.1) and RFC 7540 (8.1.2.3). Rejection of requests can be changed - to stripping the #fragment instead by setting the runtime guard ``envoy.reloadable_features.http_reject_path_with_fragment`` - to false. This behavior can further be changed to the deprecated behavior of keeping the fragment by setting the runtime guard - ``envoy.reloadable_features.http_strip_fragment_from_path_unsafe_if_disabled``. This runtime guard must only be set - to false when existing non-compliant traffic relies on #fragment in URI. When this option is enabled, Envoy request - authorization extensions may be bypassed. This override and its associated behavior will be decommissioned after the standard deprecation period. -* http: set the default :ref:`lazy headermap threshold ` to 3, - which defines the minimal number of headers in a request/response/trailers required for using a - dictionary in addition to the list. Setting the ``envoy.http.headermap.lazy_map_min_size`` runtime - feature to a non-negative number will override the default value. -* http: stop processing pending H/2 frames if connection transitioned to a closed state. This behavior can be temporarily reverted by setting the ``envoy.reloadable_features.skip_dispatching_frames_for_closed_connection`` to false. -* listener: added the :ref:`enable_reuse_port ` - field and changed the default for reuse_port from false to true, as the feature is now well - supported on the majority of production Linux kernels in use. The default change is aware of the hot - restart, as otherwise, the change would not be backward compatible between restarts. This means - that hot restarting onto a new binary will retain the default of false until the binary undergoes - a full restart. To retain the previous behavior, either explicitly set the new configuration - field to false, or set the runtime feature flag ``envoy.reloadable_features.listener_reuse_port_default_enabled`` - to false. As part of this change, the use of reuse_port for TCP listeners on both macOS and - Windows has been disabled due to suboptimal behavior. See the field documentation for more - information. -* listener: destroy per network filter chain stats when a network filter chain is removed during the listener in-place update. -* quic: enables IETF connection migration. This feature requires a stable UDP packet routine in the L4 load balancer with the same first-4-bytes in connection id. It can be turned off by setting runtime guard ``envoy.reloadable_features.FLAGS_quic_reloadable_flag_quic_connection_migration_use_new_cid_v2`` to false. +* bandwidth_limit: added :ref:`response trailers ` when request or response delay are enforced. +* bandwidth_limit: added :ref:`bandwidth limit stats ` request_enforced and response_enforced. Bug Fixes --------- *Changes expected to improve the state of the world and are unlikely to have negative effects* -* access log: fix ``%UPSTREAM_CLUSTER%`` when used in http upstream access logs. Previously, it was always logging as an unset value. -* aws request signer: fix the AWS Request Signer extension to correctly normalize the path and query string to be signed according to AWS' guidelines, so that the hash on the server side matches. See `AWS SigV4 documentation `_. -* cluster: delete pools when they're idle to fix unbounded memory use when using PROXY protocol upstream with tcp_proxy. This behavior can be temporarily reverted by setting the ``envoy.reloadable_features.conn_pool_delete_when_idle`` runtime guard to false. -* cluster: finish cluster warming even if hosts are removed before health check initialization. This only affected clusters with :ref:`ignore_health_on_host_removal `. -* compressor: fix a bug where if trailers were added and a subsequent filter paused the filter chain, the request could be stalled. This behavior can be reverted by setting ``envoy.reloadable_features.fix_added_trailers`` to false. -* dynamic forward proxy: fixing a validation bug where san and sni checks were not applied setting :ref:`http_protocol_options ` via :ref:`typed_extension_protocol_options `. -* ext_authz: fix the ext_authz filter to correctly merge multiple same headers using the ',' as separator in the check request to the external authorization service. -* ext_authz: fix the use of ``append`` field of :ref:`response_headers_to_add ` to set or append encoded response headers from a gRPC auth server. -* ext_authz: fix the HTTP ext_authz filter to respond with ``403 Forbidden`` when a gRPC auth server sends a denied check response with an empty HTTP status code. -* ext_authz: the network ext_authz filter now correctly sets dynamic metadata returned by the authorization service for non-OK responses. This behavior now matches the http ext_authz filter. -* hcm: remove deprecation for :ref:`xff_num_trusted_hops ` and forbid mixing ip detection extensions with old related knobs. -* http: limit use of deferred resets in the http2 codec to server-side connections. Use of deferred reset for client connections can result in incorrect behavior and performance problems. -* listener: fixed an issue on Windows where connections are not handled by all worker threads. -* lua: fix ``BodyBuffer`` setting a Lua string and printing Lua string containing hex characters. Previously, ``BodyBuffer`` setting a Lua string or printing strings with hex characters will be truncated. -* xray: fix the AWS X-Ray tracer bug where span's error, fault and throttle information was not reported properly as per the `AWS X-Ray documentation `_. Before this fix, server error was reported under the 'annotations' section of the segment data. +* listener: fixed the crash when updating listeners that do not bind to port. +* thrift_proxy: fix the thrift_proxy connection manager to correctly report success/error response metrics when performing :ref:`payload passthrough `. Removed Config or Runtime ------------------------- *Normally occurs at the end of the* :ref:`deprecation period ` -* http: removed ``envoy.reloadable_features.http_upstream_wait_connect_response`` runtime guard and legacy code paths. -* http: removed ``envoy.reloadable_features.allow_preconnect`` runtime guard and legacy code paths. -* listener: removed ``envoy.reloadable_features.disable_tls_inspector_injection`` runtime guard and legacy code paths. -* ocsp: removed ``envoy.reloadable_features.check_ocsp_policy deprecation`` runtime guard and legacy code paths. -* ocsp: removed ``envoy.reloadable_features.require_ocsp_response_for_must_staple_certs deprecation`` and legacy code paths. -* quic: removed ``envoy.reloadable_features.prefer_quic_kernel_bpf_packet_routing`` runtime guard. +* http: removed ``envoy.reloadable_features.return_502_for_upstream_protocol_errors``. Envoy will always return 502 code upon encountering upstream protocol error. +* http: removed ``envoy.reloadable_features.treat_upstream_connect_timeout_as_connect_failure`` and legacy code paths. New Features ------------ -* access_log: added :ref:`METADATA` token to handle all types of metadata (DYNAMIC, CLUSTER, ROUTE). -* bootstrap: added :ref:`inline_headers ` in the bootstrap to make custom inline headers bootstrap configurable. -* contrib: added new :ref:`contrib images ` which contain contrib extensions. -* dns: added :ref:`V4_PREFERRED ` option to return V6 addresses only if V4 addresses are not available. -* ext_authz: added :ref:`dynamic_metadata_from_headers ` to support emitting dynamic metadata from headers returned by an external authorization service via HTTP. -* grpc reverse bridge: added a new :ref:`option ` to support streaming response bodies when withholding gRPC frames from the upstream. -* http: added cluster_header in :ref:`weighted_clusters ` to allow routing to the weighted cluster specified in the request_header. -* http: added :ref:`alternate_protocols_cache_options ` for enabling HTTP/3 connections to servers which advertise HTTP/3 support via `HTTP Alternative Services `_. -* http: added :ref:`string_match ` in the header matcher. -* http: added :ref:`x-envoy-upstream-stream-duration-ms ` that allows configuring the max stream duration via a request header. -* http: added support for :ref:`max_requests_per_connection ` for both upstream and downstream connections. -* http: sanitizing the referer header as documented :ref:`here `. This feature can be temporarily turned off by setting runtime guard ``envoy.reloadable_features.sanitize_http_header_referer`` to false. -* http: validating outgoing HTTP/2 CONNECT requests to ensure that if ``:path`` is set that ``:protocol`` is present. This behavior can be temporarily turned off by setting runtime guard ``envoy.reloadable_features.validate_connect`` to false. -* jwt_authn: added support for :ref:`Jwt Cache ` and its size can be specified by :ref:`jwt_cache_size `. -* jwt_authn: added support for extracting JWTs from request cookies using :ref:`from_cookies `. -* jwt_authn: added support for setting the extracted headers from a successfully verified JWT using :ref:`header_in_metadata ` to dynamic metadata. -* listener: new listener metric ``downstream_cx_transport_socket_connect_timeout`` to track transport socket timeouts. -* lua: added ``header:getAtIndex()`` and ``header:getNumValues()`` methods to :ref:`header object ` for retrieving the value of a header at certain index and get the total number of values for a given header. -* matcher: added :ref:`invert ` for inverting the match result in the metadata matcher. -* overload: add a new overload action that resets streams using a lot of memory. To enable the tracking of allocated bytes in buffers that a stream is using we need to configure the minimum threshold for tracking via:ref:`buffer_factory_config `. We have an overload action ``Envoy::Server::OverloadActionNameValues::ResetStreams`` that takes advantage of the tracking to reset the most expensive stream first. -* rbac: added :ref:`destination_port_range ` for matching range of destination ports. -* route config: added :ref:`dynamic_metadata ` for routing based on dynamic metadata. -* router: added retry options predicate extensions configured via - :ref:` `. These - extensions allow modification of requests between retries at the router level. There are not - currently any built-in extensions that implement this extension point. -* router: added :ref:`per_try_idle_timeout ` timeout configuration. -* router: added an optional :ref:`override_auto_sni_header ` to support setting SNI value from an arbitrary header other than host/authority. -* sxg_filter: added filter to transform response to SXG package to :ref:`contrib images `. This can be enabled by setting :ref:`SXG ` configuration. -* thrift_proxy: added support for :ref:`mirroring requests `. -* udp: allows updating filter chain in-place through LDS, which is supported by Quic listener. Such listener config will be rejected in other connection-less UDP listener implementations. It can be reverted by ``envoy.reloadable_features.udp_listener_updates_filter_chain_in_place``. -* udp: disallow L4 filter chain in config which configures connection-less UDP listener. It can be reverted by ``envoy.reloadable_features.udp_listener_updates_filter_chain_in_place``. +* http: added support for :ref:`retriable health check status codes `. Deprecated ----------- - -* api: the :ref:`matcher ` field has been deprecated in favor of - :ref:`matcher ` in order to break a build dependency. -* cluster: :ref:`max_requests_per_connection ` is deprecated in favor of :ref:`max_requests_per_connection `. -* http: the HeaderMatcher fields :ref:`exact_match `, :ref:`safe_regex_match `, - :ref:`prefix_match `, :ref:`suffix_match ` and - :ref:`contains_match ` are deprecated by :ref:`string_match `. -* listener: :ref:`reuse_port ` has been - deprecated in favor of :ref:`enable_reuse_port `. - At the same time, the default has been changed from false to true. See above for more information. +---------- \ No newline at end of file diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc index 78363e2d99749..09a9cb5d10c7b 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc @@ -8,7 +8,7 @@ #include "source/common/http/utility.h" #include "source/common/stats/timespan_impl.h" -using envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit; +using envoy::extensions::filters::http::bandwidth_limit::v3::BandwidthLimit; using Envoy::Extensions::HttpFilters::Common::StreamRateLimiter; namespace Envoy { diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h index bf8af664e6141..9a89eb1c7f653 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h @@ -5,7 +5,7 @@ #include #include -#include "envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.pb.h" +#include "envoy/extensions/filters/http/bandwidth_limit/v3/bandwidth_limit.pb.h" #include "envoy/http/filter.h" #include "envoy/runtime/runtime.h" #include "envoy/stats/scope.h" @@ -57,10 +57,10 @@ struct BandwidthLimitStats { class FilterConfig : public ::Envoy::Router::RouteSpecificFilterConfig { public: using EnableMode = - envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit_EnableMode; + envoy::extensions::filters::http::bandwidth_limit::v3::BandwidthLimit_EnableMode; FilterConfig( - const envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit& config, + const envoy::extensions::filters::http::bandwidth_limit::v3::BandwidthLimit& config, Stats::Scope& scope, Runtime::Loader& runtime, TimeSource& time_source, bool per_route = false); ~FilterConfig() override = default; diff --git a/test/extensions/filters/http/bandwidth_limit/config_test.cc b/test/extensions/filters/http/bandwidth_limit/config_test.cc index 24635d1e130d8..af808eb9e36e3 100644 --- a/test/extensions/filters/http/bandwidth_limit/config_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/config_test.cc @@ -12,7 +12,7 @@ namespace HttpFilters { namespace BandwidthLimitFilter { using EnableMode = - envoy::extensions::filters::http::bandwidth_limit::v3alpha::BandwidthLimit_EnableMode; + envoy::extensions::filters::http::bandwidth_limit::v3::BandwidthLimit_EnableMode; TEST(Factory, GlobalEmptyConfig) { const std::string yaml = R"( diff --git a/test/extensions/filters/http/bandwidth_limit/filter_test.cc b/test/extensions/filters/http/bandwidth_limit/filter_test.cc index e943d756d14c5..c75adfc3fbc91 100644 --- a/test/extensions/filters/http/bandwidth_limit/filter_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/filter_test.cc @@ -1,4 +1,4 @@ -#include "envoy/extensions/filters/http/bandwidth_limit/v3alpha/bandwidth_limit.pb.h" +#include "envoy/extensions/filters/http/bandwidth_limit/v3/bandwidth_limit.pb.h" #include "source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h" From 70fd09be12f19046b61db815745aa969df270e68 Mon Sep 17 00:00:00 2001 From: gayang Date: Mon, 11 Oct 2021 10:22:55 +0000 Subject: [PATCH 12/18] fix current.rst merge issue Signed-off-by: gayang --- docs/root/version_history/current.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index a6bac87d48d4d..f36f94581eb21 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -29,10 +29,6 @@ Removed Config or Runtime New Features ------------ -* http: added support for :ref:`retriable health check status codes `. - -Deprecated ----------- * ext_authz: added :ref:`query_parameters_to_set ` and :ref:`query_parameters_to_remove ` for adding and removing query string parameters when using a gRPC authorization server. * http: added support for :ref:`retriable health check status codes `. From 445f82d40269008cc89b87f5e19c7d3aff54e6d9 Mon Sep 17 00:00:00 2001 From: gayang Date: Tue, 12 Oct 2021 01:41:58 +0000 Subject: [PATCH 13/18] update api comments Signed-off-by: gayang --- .../filters/http/bandwidth_limit/v3/bandwidth_limit.proto | 2 +- docs/root/version_history/current.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/envoy/extensions/filters/http/bandwidth_limit/v3/bandwidth_limit.proto b/api/envoy/extensions/filters/http/bandwidth_limit/v3/bandwidth_limit.proto index 9eb1cd7b20a1e..8c4d1b7b208ca 100644 --- a/api/envoy/extensions/filters/http/bandwidth_limit/v3/bandwidth_limit.proto +++ b/api/envoy/extensions/filters/http/bandwidth_limit/v3/bandwidth_limit.proto @@ -75,7 +75,7 @@ message BandwidthLimit { // * bandwidth-request-delay-ms: delay time in milliseconds it took for the request stream transfer. // * bandwidth-response-delay-ms: delay time in milliseconds it took for the response stream transfer. // * If :ref:`enable_mode ` is DISABLED or REQUEST, the trailers will not be set. - // * If the request/response delay time is 0, the trailers will not be set. + // * If both the request and response delay time is 0, the trailers will not be set. // bool enable_response_trailers = 6; diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index f36f94581eb21..ceb52b853f694 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -10,7 +10,7 @@ Minor Behavior Changes *Changes that may cause incompatibilities for some users, but should not for most* * bandwidth_limit: added :ref:`response trailers ` when request or response delay are enforced. -* bandwidth_limit: added :ref:`bandwidth limit stats ` request_enforced and response_enforced. +* bandwidth_limit: added :ref:`bandwidth limit stats ` *request_enforced* and *response_enforced*. Bug Fixes --------- From ff204cc03b14d6df04e1ca1c38e5f1a8ae06c17c Mon Sep 17 00:00:00 2001 From: gayang Date: Thu, 21 Oct 2021 07:01:23 +0000 Subject: [PATCH 14/18] fix comments Signed-off-by: gayang --- .../http/bandwidth_limit/bandwidth_limit.cc | 34 +++++++++++-------- .../http/bandwidth_limit/bandwidth_limit.h | 2 +- .../http/bandwidth_limit/config_test.cc | 5 ++- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc index 09a9cb5d10c7b..607be12099025 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc @@ -21,6 +21,7 @@ const Http::LowerCaseString DefaultRequestDelayTrailer = Http::LowerCaseString("bandwidth-request-delay-ms"); const Http::LowerCaseString DefaultResponseDelayTrailer = Http::LowerCaseString("bandwidth-response-delay-ms"); +const std::chrono::milliseconds ZeroMilliseconds = std::chrono::milliseconds(0); } // namespace FilterConfig::FilterConfig(const BandwidthLimit& config, Stats::Scope& scope, @@ -31,14 +32,16 @@ FilterConfig::FilterConfig(const BandwidthLimit& config, Stats::Scope& scope, config, fill_interval, StreamRateLimiter::DefaultFillInterval.count()))), enabled_(config.runtime_enabled(), runtime), stats_(generateStats(config.stat_prefix(), scope)), - request_delay_trailer_(config.response_trailer_prefix().empty() - ? DefaultRequestDelayTrailer - : Http::LowerCaseString(config.response_trailer_prefix() + "-" + - DefaultRequestDelayTrailer.get())), - response_delay_trailer_(config.response_trailer_prefix().empty() - ? DefaultResponseDelayTrailer - : Http::LowerCaseString(config.response_trailer_prefix() + "-" + - DefaultResponseDelayTrailer.get())), + request_delay_trailer_( + config.response_trailer_prefix().empty() + ? DefaultRequestDelayTrailer + : Http::LowerCaseString(absl::StrCat(config.response_trailer_prefix(), "-", + DefaultRequestDelayTrailer.get()))), + response_delay_trailer_( + config.response_trailer_prefix().empty() + ? DefaultResponseDelayTrailer + : Http::LowerCaseString(absl::StrCat(config.response_trailer_prefix(), "-", + DefaultResponseDelayTrailer.get()))), enable_response_trailers_(config.enable_response_trailers()) { if (per_route && !config.has_limit_kbps()) { throw EnvoyException("bandwidthlimitfilter: limit must be set for per route filter config"); @@ -164,7 +167,7 @@ Http::FilterDataStatus BandwidthLimiter::encodeData(Buffer::Instance& data, bool // Adds encoded trailers. May only be called in encodeData when end_stream is set to true. // If upstream has trailers, addEncodedTrailers won't be called bool trailer_added = false; - if (end_stream) { + if (end_stream && config.enableResponseTrailers()) { trailers_ = &encoder_callbacks_->addEncodedTrailers(); trailer_added = true; } @@ -201,7 +204,7 @@ BandwidthLimiter::encodeTrailers(Http::ResponseTrailerMap& response_trailers) { void BandwidthLimiter::updateStatsOnDecodeFinish() { if (request_latency_) { - request_duration_ = request_latency_.get()->elapsed().count(); + request_duration_ = request_latency_.get()->elapsed(); request_latency_->complete(); request_latency_.reset(); getConfig().stats().request_pending_.dec(); @@ -213,12 +216,13 @@ void BandwidthLimiter::updateStatsOnEncodeFinish() { const auto& config = getConfig(); if (config.enableResponseTrailers() && trailers_ != nullptr) { - auto response_duration = response_latency_.get()->elapsed().count(); - if (request_duration_ > 0) { - trailers_->setCopy(config.requestDelayTrailer(), std::to_string(request_duration_)); + auto response_duration = response_latency_.get()->elapsed(); + if (request_duration_ > ZeroMilliseconds) { + trailers_->setCopy(config.requestDelayTrailer(), std::to_string(request_duration_.count())); } - if (response_duration > 0) { - trailers_->setCopy(config.responseDelayTrailer(), std::to_string(response_duration)); + if (response_duration > ZeroMilliseconds) { + trailers_->setCopy(config.responseDelayTrailer(), + std::to_string(response_duration.count())); } } diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h index 6c30ffb112923..a0d0612d5fdcc 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h @@ -149,7 +149,7 @@ class BandwidthLimiter : public Http::StreamFilter, Logger::Loggable response_limiter_; Stats::TimespanPtr request_latency_; Stats::TimespanPtr response_latency_; - uint64_t request_duration_ = 0; + std::chrono::milliseconds request_duration_; Http::ResponseTrailerMap* trailers_; }; diff --git a/test/extensions/filters/http/bandwidth_limit/config_test.cc b/test/extensions/filters/http/bandwidth_limit/config_test.cc index 0709b128ad4ec..c7b66374d8f9b 100644 --- a/test/extensions/filters/http/bandwidth_limit/config_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/config_test.cc @@ -37,6 +37,7 @@ TEST(Factory, RouteSpecificFilterConfig) { enable_mode: REQUEST_AND_RESPONSE limit_kbps: 10 fill_interval: 0.1s + enable_response_trailers: true response_trailer_prefix: test )"; @@ -54,6 +55,7 @@ TEST(Factory, RouteSpecificFilterConfig) { EXPECT_EQ(config->fillInterval().count(), 100); EXPECT_EQ(config->enableMode(), EnableMode::BandwidthLimit_EnableMode_REQUEST_AND_RESPONSE); EXPECT_FALSE(config->tokenBucket() == nullptr); + EXPECT_EQ(config->enableResponseTrailers(), true); EXPECT_EQ(const_cast(config)->requestDelayTrailer(), Http::LowerCaseString("test-bandwidth-request-delay-ms")); EXPECT_EQ(const_cast(config)->responseDelayTrailer(), @@ -82,7 +84,7 @@ TEST(Factory, RouteSpecificFilterConfigDisabledByDefault) { EXPECT_EQ(config->fillInterval().count(), 100); } -TEST(Factory, RouteSpecificFilterConfigDefaultFillInterval) { +TEST(Factory, RouteSpecificFilterConfigDefault) { const std::string config_yaml = R"( stat_prefix: test enable_mode: REQUEST_AND_RESPONSE @@ -102,6 +104,7 @@ TEST(Factory, RouteSpecificFilterConfigDefaultFillInterval) { EXPECT_EQ(config->limit(), 10); EXPECT_EQ(config->fillInterval().count(), 50); // default trailers + EXPECT_EQ(config->enableResponseTrailers(), false); EXPECT_EQ(const_cast(config)->requestDelayTrailer(), Http::LowerCaseString("bandwidth-request-delay-ms")); EXPECT_EQ(const_cast(config)->responseDelayTrailer(), From 36443a1639aaaf1e3733cbe8301617cede047ebd Mon Sep 17 00:00:00 2001 From: gayang Date: Thu, 21 Oct 2021 07:23:31 +0000 Subject: [PATCH 15/18] change metrics request_incoming_size, request_allowed_size, response_incoming_size, response_allowed_size back to gauge Signed-off-by: gayang --- .../http/http_filters/bandwidth_limit_filter.rst | 8 ++++---- .../filters/http/bandwidth_limit/bandwidth_limit.cc | 8 ++++---- .../filters/http/bandwidth_limit/bandwidth_limit.h | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst b/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst index 954c1b74528c0..57072ac10a8de 100644 --- a/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst +++ b/docs/root/configuration/http/http_filters/bandwidth_limit_filter.rst @@ -44,14 +44,14 @@ The HTTP bandwidth limit filter outputs statistics in the ``.http_b request_enabled, Counter, Total number of request streams for which the bandwidth limiter was consulted request_enforced, Counter, Total number of request streams for which the bandwidth limiter was enforced request_pending, GAUGE, Number of request streams which are currently pending transfer in bandwidth limiter - request_incoming_size, Counter, Size in bytes of incoming request data to bandwidth limiter - request_allowed_size, Counter, Size in bytes of outgoing request data from bandwidth limiter + request_incoming_size, GAUGE, Size in bytes of incoming request data to bandwidth limiter + request_allowed_size, GAUGE, Size in bytes of outgoing request data from bandwidth limiter request_transfer_duration, HISTOGRAM, Total time (including added delay) it took for the request stream transfer response_enabled, Counter, Total number of response streams for which the bandwidth limiter was consulted response_enforced, Counter, Total number of response streams for which the bandwidth limiter was enforced response_pending, GAUGE, Number of response streams which are currently pending transfer in bandwidth limiter - response_incoming_size, Counter, Size in bytes of incoming response data to bandwidth limiter - response_allowed_size, Counter, Size in bytes of outgoing response data from bandwidth limiter + response_incoming_size, GAUGE, Size in bytes of incoming response data to bandwidth limiter + response_allowed_size, GAUGE, Size in bytes of outgoing response data from bandwidth limiter response_transfer_duration, HISTOGRAM, Total time (including added delay) it took for the response stream transfer .. _config_http_filters_bandwidth_limit_runtime: diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc index 607be12099025..2e7178d5a26ab 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.cc @@ -84,7 +84,7 @@ Http::FilterHeadersStatus BandwidthLimiter::decodeHeaders(Http::RequestHeaderMap decoder_callbacks_->continueDecoding(); }, [&config](uint64_t len, bool limit_enforced) { - config.stats().request_allowed_size_.add(len); + config.stats().request_allowed_size_.set(len); if (limit_enforced) { config.stats().request_enforced_.inc(); } @@ -106,7 +106,7 @@ Http::FilterDataStatus BandwidthLimiter::decodeData(Buffer::Instance& data, bool const_cast(&config)->timeSource()); config.stats().request_pending_.inc(); } - config.stats().request_incoming_size_.add(data.length()); + config.stats().request_incoming_size_.set(data.length()); request_limiter_->writeData(data, end_stream); return Http::FilterDataStatus::StopIterationNoBuffer; @@ -148,7 +148,7 @@ Http::FilterHeadersStatus BandwidthLimiter::encodeHeaders(Http::ResponseHeaderMa encoder_callbacks_->continueEncoding(); }, [&config](uint64_t len, bool limit_enforced) { - config.stats().response_allowed_size_.add(len); + config.stats().response_allowed_size_.set(len); if (limit_enforced) { config.stats().response_enforced_.inc(); } @@ -178,7 +178,7 @@ Http::FilterDataStatus BandwidthLimiter::encodeData(Buffer::Instance& data, bool const_cast(&config)->timeSource()); config.stats().response_pending_.inc(); } - config.stats().response_incoming_size_.add(data.length()); + config.stats().response_incoming_size_.set(data.length()); response_limiter_->writeData(data, end_stream, trailer_added); return Http::FilterDataStatus::StopIterationNoBuffer; diff --git a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h index a0d0612d5fdcc..c7c2242a0258c 100644 --- a/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h +++ b/source/extensions/filters/http/bandwidth_limit/bandwidth_limit.h @@ -36,10 +36,10 @@ namespace BandwidthLimitFilter { COUNTER(response_enforced) \ GAUGE(request_pending, Accumulate) \ GAUGE(response_pending, Accumulate) \ - COUNTER(request_incoming_size) \ - COUNTER(response_incoming_size) \ - COUNTER(request_allowed_size) \ - COUNTER(response_allowed_size) \ + GAUGE(request_incoming_size, Accumulate) \ + GAUGE(response_incoming_size, Accumulate) \ + GAUGE(request_allowed_size, Accumulate) \ + GAUGE(response_allowed_size, Accumulate) \ HISTOGRAM(request_transfer_duration, Milliseconds) \ HISTOGRAM(response_transfer_duration, Milliseconds) From e29061d83a06c03bf9afdbd2be17c47610a9fac9 Mon Sep 17 00:00:00 2001 From: gayang Date: Fri, 22 Oct 2021 02:08:46 +0000 Subject: [PATCH 16/18] triger build Signed-off-by: gayang --- test/extensions/filters/http/bandwidth_limit/config_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/extensions/filters/http/bandwidth_limit/config_test.cc b/test/extensions/filters/http/bandwidth_limit/config_test.cc index c7b66374d8f9b..6f3e51c510bf4 100644 --- a/test/extensions/filters/http/bandwidth_limit/config_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/config_test.cc @@ -103,6 +103,7 @@ TEST(Factory, RouteSpecificFilterConfigDefault) { const auto* config = dynamic_cast(route_config.get()); EXPECT_EQ(config->limit(), 10); EXPECT_EQ(config->fillInterval().count(), 50); + // default trailers EXPECT_EQ(config->enableResponseTrailers(), false); EXPECT_EQ(const_cast(config)->requestDelayTrailer(), From fbd36a054b958c4130719f70942390ac4a251f6b Mon Sep 17 00:00:00 2001 From: gayang Date: Fri, 22 Oct 2021 02:48:18 +0000 Subject: [PATCH 17/18] fix format Signed-off-by: gayang --- test/extensions/filters/http/bandwidth_limit/config_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions/filters/http/bandwidth_limit/config_test.cc b/test/extensions/filters/http/bandwidth_limit/config_test.cc index 6f3e51c510bf4..a3b9334e2e8be 100644 --- a/test/extensions/filters/http/bandwidth_limit/config_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/config_test.cc @@ -103,7 +103,7 @@ TEST(Factory, RouteSpecificFilterConfigDefault) { const auto* config = dynamic_cast(route_config.get()); EXPECT_EQ(config->limit(), 10); EXPECT_EQ(config->fillInterval().count(), 50); - + // default trailers EXPECT_EQ(config->enableResponseTrailers(), false); EXPECT_EQ(const_cast(config)->requestDelayTrailer(), From b9cea1d45f612c0046f4abb7bc715c21f430ed94 Mon Sep 17 00:00:00 2001 From: gayang Date: Fri, 22 Oct 2021 03:25:27 +0000 Subject: [PATCH 18/18] fix tests Signed-off-by: gayang --- .../http/bandwidth_limit/config_test.cc | 1 - .../http/bandwidth_limit/filter_test.cc | 44 +++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/test/extensions/filters/http/bandwidth_limit/config_test.cc b/test/extensions/filters/http/bandwidth_limit/config_test.cc index a3b9334e2e8be..c7b66374d8f9b 100644 --- a/test/extensions/filters/http/bandwidth_limit/config_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/config_test.cc @@ -103,7 +103,6 @@ TEST(Factory, RouteSpecificFilterConfigDefault) { const auto* config = dynamic_cast(route_config.get()); EXPECT_EQ(config->limit(), 10); EXPECT_EQ(config->fillInterval().count(), 50); - // default trailers EXPECT_EQ(config->enableResponseTrailers(), false); EXPECT_EQ(const_cast(config)->requestDelayTrailer(), diff --git a/test/extensions/filters/http/bandwidth_limit/filter_test.cc b/test/extensions/filters/http/bandwidth_limit/filter_test.cc index c18df6ec04e06..338c53ffa6c40 100644 --- a/test/extensions/filters/http/bandwidth_limit/filter_test.cc +++ b/test/extensions/filters/http/bandwidth_limit/filter_test.cc @@ -115,12 +115,12 @@ TEST_F(FilterTest, LimitOnDecode) { EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data1, false)); EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); - EXPECT_EQ(5, findCounter("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.request_incoming_size")); EXPECT_CALL(decoder_filter_callbacks_, injectDecodedDataToFilterChain(BufferStringEqual("hello"), false)); token_timer->invokeCallback(); EXPECT_EQ(0, findCounter("test.http_bandwidth_limit.request_enforced")); - EXPECT_EQ(5, findCounter("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.request_allowed_size")); // Advance time by 1s which should refill all tokens. time_system_.advanceTimeWait(std::chrono::seconds(1)); @@ -131,7 +131,7 @@ TEST_F(FilterTest, LimitOnDecode) { Buffer::OwnedImpl data2(std::string(1126, 'a')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data2, false)); EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); - EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.request_incoming_size")); EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(50), _)); EXPECT_CALL(decoder_filter_callbacks_, onDecoderFilterBelowWriteBufferLowWatermark()); @@ -139,8 +139,8 @@ TEST_F(FilterTest, LimitOnDecode) { injectDecodedDataToFilterChain(BufferStringEqual(std::string(1024, 'a')), false)); token_timer->invokeCallback(); EXPECT_EQ(1, findCounter("test.http_bandwidth_limit.request_enforced")); - EXPECT_EQ(1029, findCounter("test.http_bandwidth_limit.request_allowed_size")); - EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.request_incoming_size")); // Fire timer, also advance time. time_system_.advanceTimeWait(std::chrono::milliseconds(50)); @@ -149,14 +149,14 @@ TEST_F(FilterTest, LimitOnDecode) { injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); token_timer->invokeCallback(); EXPECT_EQ(2, findCounter("test.http_bandwidth_limit.request_enforced")); - EXPECT_EQ(1080, findCounter("test.http_bandwidth_limit.request_allowed_size")); - EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.request_incoming_size")); // Get new data with current data buffered, not end_stream. Buffer::OwnedImpl data3(std::string(51, 'b')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data3, false)); EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.request_pending")); - EXPECT_EQ(1182, findCounter("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_incoming_size")); // Fire timer, also advance time. time_system_.advanceTimeWait(std::chrono::milliseconds(50)); @@ -165,7 +165,7 @@ TEST_F(FilterTest, LimitOnDecode) { injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); token_timer->invokeCallback(); EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.request_enforced")); - EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_allowed_size")); // Fire timer, also advance time. No timer enable because there is nothing // buffered. @@ -174,7 +174,7 @@ TEST_F(FilterTest, LimitOnDecode) { injectDecodedDataToFilterChain(BufferStringEqual(std::string(51, 'b')), false)); token_timer->invokeCallback(); EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.request_enforced")); - EXPECT_EQ(1182, findCounter("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.request_allowed_size")); // Advance time by 1s for a full refill. time_system_.advanceTimeWait(std::chrono::seconds(1)); @@ -184,12 +184,12 @@ TEST_F(FilterTest, LimitOnDecode) { EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); Buffer::OwnedImpl data4(std::string(1024, 'c')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->decodeData(data4, true)); - EXPECT_EQ(2206, findCounter("test.http_bandwidth_limit.request_incoming_size")); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.request_incoming_size")); EXPECT_CALL(decoder_filter_callbacks_, injectDecodedDataToFilterChain(BufferStringEqual(std::string(1024, 'c')), true)); token_timer->invokeCallback(); EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.request_enforced")); - EXPECT_EQ(2206, findCounter("test.http_bandwidth_limit.request_allowed_size")); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.request_allowed_size")); EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.request_pending")); EXPECT_EQ(false, response_trailers_.has("test-bandwidth-request-delay-ms")); EXPECT_EQ(false, response_trailers_.has("test-bandwidth-response-delay-ms")); @@ -230,12 +230,12 @@ TEST_F(FilterTest, LimitOnEncode) { EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data1, false)); EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); - EXPECT_EQ(5, findCounter("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.response_incoming_size")); EXPECT_CALL(encoder_filter_callbacks_, injectEncodedDataToFilterChain(BufferStringEqual("hello"), false)); token_timer->invokeCallback(); EXPECT_EQ(0, findCounter("test.http_bandwidth_limit.response_enforced")); - EXPECT_EQ(5, findCounter("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(5, findGauge("test.http_bandwidth_limit.response_allowed_size")); // Advance time by 1s which should refill all tokens. time_system_.advanceTimeWait(std::chrono::seconds(1)); @@ -245,7 +245,7 @@ TEST_F(FilterTest, LimitOnEncode) { EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); Buffer::OwnedImpl data2(std::string(1126, 'a')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data2, false)); - EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.response_incoming_size")); EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(50), _)); EXPECT_CALL(encoder_filter_callbacks_, onEncoderFilterBelowWriteBufferLowWatermark()); @@ -254,8 +254,8 @@ TEST_F(FilterTest, LimitOnEncode) { token_timer->invokeCallback(); EXPECT_EQ(1, findGauge("test.http_bandwidth_limit.response_pending")); EXPECT_EQ(1, findCounter("test.http_bandwidth_limit.response_enforced")); - EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.response_incoming_size")); - EXPECT_EQ(1029, findCounter("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(1126, findGauge("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.response_allowed_size")); // Fire timer, also advance time. time_system_.advanceTimeWait(std::chrono::milliseconds(50)); @@ -264,7 +264,7 @@ TEST_F(FilterTest, LimitOnEncode) { injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); token_timer->invokeCallback(); EXPECT_EQ(2, findCounter("test.http_bandwidth_limit.response_enforced")); - EXPECT_EQ(1080, findCounter("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.response_allowed_size")); // Get new data with current data buffered, not end_stream. Buffer::OwnedImpl data3(std::string(51, 'b')); @@ -277,7 +277,7 @@ TEST_F(FilterTest, LimitOnEncode) { injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'a')), false)); token_timer->invokeCallback(); EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.response_enforced")); - EXPECT_EQ(1131, findCounter("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.response_allowed_size")); // Fire timer, also advance time. No time enable because there is nothing // buffered. @@ -286,7 +286,7 @@ TEST_F(FilterTest, LimitOnEncode) { injectEncodedDataToFilterChain(BufferStringEqual(std::string(51, 'b')), false)); token_timer->invokeCallback(); EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.response_enforced")); - EXPECT_EQ(1182, findCounter("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(51, findGauge("test.http_bandwidth_limit.response_allowed_size")); // Advance time by 1s for a full refill. time_system_.advanceTimeWait(std::chrono::seconds(1)); @@ -296,13 +296,13 @@ TEST_F(FilterTest, LimitOnEncode) { EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); Buffer::OwnedImpl data4(std::string(1024, 'c')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data4, true)); - EXPECT_EQ(2206, findCounter("test.http_bandwidth_limit.response_incoming_size")); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.response_incoming_size")); EXPECT_CALL(encoder_filter_callbacks_, injectEncodedDataToFilterChain(BufferStringEqual(std::string(1024, 'c')), false)); token_timer->invokeCallback(); EXPECT_EQ(0, findGauge("test.http_bandwidth_limit.response_pending")); EXPECT_EQ(3, findCounter("test.http_bandwidth_limit.response_enforced")); - EXPECT_EQ(2206, findCounter("test.http_bandwidth_limit.response_allowed_size")); + EXPECT_EQ(1024, findGauge("test.http_bandwidth_limit.response_allowed_size")); EXPECT_EQ(false, response_trailers_.has("test-bandwidth-request-delay-ms")); EXPECT_EQ("2150", trailers_.get_("test-bandwidth-response-delay-ms"));