From 85387923d2c800c793be9bdf959d55c3a22363a0 Mon Sep 17 00:00:00 2001 From: Tarun Sharma Date: Fri, 19 Nov 2021 13:05:39 +0530 Subject: [PATCH 01/51] cleanup: replace std::string to absl::string (#19032) Additional Description: See #11318 for details. Risk Level: N/A Testing: N/A Docs Changes: N/A Release Notes: N/A Platform Specific Features: N/A Signed-off-by: Tarun Sharma --- source/common/common/base64.cc | 4 ++-- source/common/common/base64.h | 4 ++-- source/common/config/grpc_mux_impl.cc | 2 +- source/common/config/grpc_mux_impl.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/common/base64.cc b/source/common/common/base64.cc index 9eab86b1c48bc..e719eb9a48630 100644 --- a/source/common/common/base64.cc +++ b/source/common/common/base64.cc @@ -143,7 +143,7 @@ inline void encodeLast(uint64_t pos, uint8_t last_char, std::string& ret, } // namespace -std::string Base64::decode(const std::string& input) { +std::string Base64::decode(absl::string_view input) { if (input.length() % 4) { return EMPTY_STRING; } @@ -242,7 +242,7 @@ void Base64::completePadding(std::string& encoded) { } } -std::string Base64Url::decode(const std::string& input) { +std::string Base64Url::decode(absl::string_view input) { if (input.empty()) { return EMPTY_STRING; } diff --git a/source/common/common/base64.h b/source/common/common/base64.h index a69ffbf910a3b..ed2181d79b663 100644 --- a/source/common/common/base64.h +++ b/source/common/common/base64.h @@ -44,7 +44,7 @@ class Base64 { * Note, decoded string may contain '\0' at any position, it should be treated as a sequence of * bytes. */ - static std::string decode(const std::string& input); + static std::string decode(absl::string_view input); /** * Base64 decode an input string. Padding is not required. @@ -82,7 +82,7 @@ class Base64Url { * Note, decoded string may contain '\0' at any position, it should be treated as a sequence of * bytes. */ - static std::string decode(const std::string& input); + static std::string decode(absl::string_view input); }; } // namespace Envoy diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 4242599901a77..f886e9019d641 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -64,7 +64,7 @@ void GrpcMuxImpl::onDynamicContextUpdate(absl::string_view resource_type_url) { void GrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } -void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { +void GrpcMuxImpl::sendDiscoveryRequest(absl::string_view type_url) { if (shutdown_) { return; } diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index e9d1a61828c7a..7ae805663fc14 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -84,7 +84,7 @@ class GrpcMuxImpl : public GrpcMux, private: void drainRequests(); void setRetryTimer(); - void sendDiscoveryRequest(const std::string& type_url); + void sendDiscoveryRequest(absl::string_view type_url); struct GrpcMuxWatchImpl : public GrpcMuxWatch { GrpcMuxWatchImpl(const absl::flat_hash_set& resources, From eda54585c5f63a721f8abf05d1603b7e263f740d Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Fri, 19 Nov 2021 20:19:52 +0000 Subject: [PATCH 02/51] Remove unused test dependencies (#19054) Signed-off-by: Yan Avlasov --- test/server/BUILD | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/server/BUILD b/test/server/BUILD index ff737d73f7808..3060862c1ef70 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -436,8 +436,6 @@ envoy_cc_test( "//source/extensions/filters/http/health_check:config", "//source/extensions/filters/http/router:config", "//source/extensions/filters/network/http_connection_manager:config", - "//source/extensions/filters/network/redis_proxy:config", - "//source/extensions/stat_sinks/statsd:config", "//source/extensions/tracers/zipkin:config", "//source/server:process_context_lib", "//source/server:server_lib", From 7dec99d8c932d5cb76c518d384917b07edfa22e4 Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Fri, 19 Nov 2021 14:58:46 -0800 Subject: [PATCH 03/51] HTTP1: Refactor HTTP1 Active Request to be defer deletable. (#19062) Signed-off-by: Kevin Baichoo --- source/common/http/http1/codec_impl.cc | 90 +++++++++++------------ source/common/http/http1/codec_impl.h | 9 ++- test/common/http/codec_impl_fuzz_test.cc | 4 + test/common/http/http1/codec_impl_test.cc | 6 ++ test/integration/fake_upstream.cc | 8 +- 5 files changed, 62 insertions(+), 55 deletions(-) diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index 0468bdbcb86fd..fdd317babbbfc 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -996,18 +996,17 @@ ServerConnectionImpl::ServerConnectionImpl( uint32_t ServerConnectionImpl::getHeadersSize() { // Add in the size of the request URL if processing request headers. - const uint32_t url_size = (!processing_trailers_ && active_request_.has_value()) - ? active_request_.value().request_url_.size() - : 0; + const uint32_t url_size = + (!processing_trailers_ && active_request_) ? active_request_->request_url_.size() : 0; return url_size + ConnectionImpl::getHeadersSize(); } void ServerConnectionImpl::onEncodeComplete() { - if (active_request_.value().remote_complete_) { + if (active_request_->remote_complete_) { // Only do this if remote is complete. If we are replying before the request is complete the // only logical thing to do is for higher level code to reset() / close the connection so we // leave the request around so that it can fire reset callbacks. - active_request_.reset(); + connection_.dispatcher().deferredDelete(std::move(active_request_)); } } @@ -1018,12 +1017,11 @@ Status ServerConnectionImpl::handlePath(RequestHeaderMap& headers, absl::string_ bool is_connect = (method == header_values.MethodValues.Connect); // The url is relative or a wildcard when the method is OPTIONS. Nothing to do here. - auto& active_request = active_request_.value(); - if (!is_connect && !active_request.request_url_.getStringView().empty() && - (active_request.request_url_.getStringView()[0] == '/' || + if (!is_connect && !active_request_->request_url_.getStringView().empty() && + (active_request_->request_url_.getStringView()[0] == '/' || (method == header_values.MethodValues.Options && - active_request.request_url_.getStringView()[0] == '*'))) { - headers.addViaMove(std::move(path), std::move(active_request.request_url_)); + active_request_->request_url_.getStringView()[0] == '*'))) { + headers.addViaMove(std::move(path), std::move(active_request_->request_url_)); return okStatus(); } @@ -1032,12 +1030,12 @@ Status ServerConnectionImpl::handlePath(RequestHeaderMap& headers, absl::string_ // CONNECT "urls" are actually host:port so look like absolute URLs to the above checks. // Absolute URLS in CONNECT requests will be rejected below by the URL class validation. if (!codec_settings_.allow_absolute_url_ && !is_connect) { - headers.addViaMove(std::move(path), std::move(active_request.request_url_)); + headers.addViaMove(std::move(path), std::move(active_request_->request_url_)); return okStatus(); } Utility::Url absolute_url; - if (!absolute_url.initialize(active_request.request_url_.getStringView(), is_connect)) { + if (!absolute_url.initialize(active_request_->request_url_.getStringView(), is_connect)) { RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().InvalidUrl)); return codecProtocolError("http/1.1 protocol error: invalid url in request line"); } @@ -1068,7 +1066,7 @@ Status ServerConnectionImpl::handlePath(RequestHeaderMap& headers, absl::string_ if (!absolute_url.pathAndQueryParams().empty()) { headers.setPath(absolute_url.pathAndQueryParams()); } - active_request.request_url_.clear(); + active_request_->request_url_.clear(); return okStatus(); } @@ -1076,8 +1074,7 @@ Envoy::StatusOr ServerConnectionImpl::onHeadersCompleteBase() { // Handle the case where response happens prior to request complete. It's up to upper layer code // to disconnect the connection but we shouldn't fire any more events since it doesn't make // sense. - if (active_request_.has_value()) { - auto& active_request = active_request_.value(); + if (active_request_) { auto& headers = absl::get(headers_or_trailers_); ENVOY_CONN_LOG(trace, "Server: onHeadersComplete size={}", connection_, headers->size()); @@ -1097,13 +1094,13 @@ Envoy::StatusOr ServerConnectionImpl::onHeadersCompleteBase() { // Inform the response encoder about any HEAD method, so it can set content // length and transfer encoding headers correctly. const Http::HeaderValues& header_values = Http::Headers::get(); - active_request.response_encoder_.setIsResponseToHeadRequest(parser_->methodName() == - header_values.MethodValues.Head); - active_request.response_encoder_.setIsResponseToConnectRequest( + active_request_->response_encoder_.setIsResponseToHeadRequest(parser_->methodName() == + header_values.MethodValues.Head); + active_request_->response_encoder_.setIsResponseToConnectRequest( parser_->methodName() == header_values.MethodValues.Connect); RETURN_IF_ERROR(handlePath(*headers, parser_->methodName())); - ASSERT(active_request.request_url_.empty()); + ASSERT(active_request_->request_url_.empty()); headers->setMethod(parser_->methodName()); @@ -1124,7 +1121,7 @@ Envoy::StatusOr ServerConnectionImpl::onHeadersCompleteBase() { if (parser_->isChunked() || (parser_->contentLength().has_value() && parser_->contentLength().value() > 0) || handling_upgrade_) { - active_request.request_decoder_->decodeHeaders(std::move(headers), false); + active_request_->request_decoder_->decodeHeaders(std::move(headers), false); // If the connection has been closed (or is closing) after decoding headers, pause the parser // so we return control to the caller. @@ -1141,13 +1138,12 @@ Envoy::StatusOr ServerConnectionImpl::onHeadersCompleteBase() { Status ServerConnectionImpl::onMessageBeginBase() { if (!resetStreamCalled()) { - ASSERT(!active_request_.has_value()); - active_request_.emplace(*this, std::move(bytes_meter_before_stream_)); - auto& active_request = active_request_.value(); + ASSERT(active_request_ == nullptr); + active_request_ = std::make_unique(*this, std::move(bytes_meter_before_stream_)); if (resetStreamCalled()) { return codecClientError("cannot create new streams after calling reset"); } - active_request.request_decoder_ = &callbacks_.newStream(active_request.response_encoder_); + active_request_->request_decoder_ = &callbacks_.newStream(active_request_->response_encoder_); // Check for pipelined request flood as we prepare to accept a new request. // Parse errors that happen prior to onMessageBegin result in stream termination, it is not @@ -1158,8 +1154,8 @@ Status ServerConnectionImpl::onMessageBeginBase() { } Status ServerConnectionImpl::onUrl(const char* data, size_t length) { - if (active_request_.has_value()) { - active_request_.value().request_url_.append(data, length); + if (active_request_) { + active_request_->request_url_.append(data, length); RETURN_IF_ERROR(checkMaxHeadersSize()); } @@ -1169,31 +1165,31 @@ Status ServerConnectionImpl::onUrl(const char* data, size_t length) { void ServerConnectionImpl::onBody(Buffer::Instance& data) { ASSERT(!deferred_end_stream_headers_); - if (active_request_.has_value()) { + if (active_request_) { ENVOY_CONN_LOG(trace, "body size={}", connection_, data.length()); - active_request_.value().request_decoder_->decodeData(data, false); + active_request_->request_decoder_->decodeData(data, false); } } ParserStatus ServerConnectionImpl::onMessageCompleteBase() { ASSERT(!handling_upgrade_); - if (active_request_.has_value()) { - auto& active_request = active_request_.value(); + if (active_request_) { + + // The request_decoder should be non-null after we've called the newStream on callbacks. + ASSERT(active_request_->request_decoder_); + active_request_->response_encoder_.readDisable(true); + active_request_->remote_complete_ = true; - if (active_request.request_decoder_) { - active_request.response_encoder_.readDisable(true); - } - active_request.remote_complete_ = true; if (deferred_end_stream_headers_) { - active_request.request_decoder_->decodeHeaders( + active_request_->request_decoder_->decodeHeaders( std::move(absl::get(headers_or_trailers_)), true); deferred_end_stream_headers_ = false; } else if (processing_trailers_) { - active_request.request_decoder_->decodeTrailers( + active_request_->request_decoder_->decodeTrailers( std::move(absl::get(headers_or_trailers_))); } else { Buffer::OwnedImpl buffer; - active_request.request_decoder_->decodeData(buffer, true); + active_request_->request_decoder_->decodeData(buffer, true); } // Reset to ensure no information from one requests persists to the next. @@ -1207,19 +1203,19 @@ ParserStatus ServerConnectionImpl::onMessageCompleteBase() { } void ServerConnectionImpl::onResetStream(StreamResetReason reason) { - active_request_.value().response_encoder_.runResetCallbacks(reason); - active_request_.reset(); + active_request_->response_encoder_.runResetCallbacks(reason); + connection_.dispatcher().deferredDelete(std::move(active_request_)); } Status ServerConnectionImpl::sendProtocolError(absl::string_view details) { // We do this here because we may get a protocol error before we have a logical stream. - if (!active_request_.has_value()) { + if (active_request_ == nullptr) { RETURN_IF_ERROR(onMessageBegin()); } - ASSERT(active_request_.has_value()); + ASSERT(active_request_); - active_request_.value().response_encoder_.setDetails(details); - if (!active_request_.value().response_encoder_.startedResponse()) { + active_request_->response_encoder_.setDetails(details); + if (!active_request_->response_encoder_.startedResponse()) { active_request_->request_decoder_->sendLocalReply( error_code_, CodeUtility::toString(error_code_), nullptr, absl::nullopt, details); } @@ -1227,13 +1223,13 @@ Status ServerConnectionImpl::sendProtocolError(absl::string_view details) { } void ServerConnectionImpl::onAboveHighWatermark() { - if (active_request_.has_value()) { - active_request_.value().response_encoder_.runHighWatermarkCallbacks(); + if (active_request_) { + active_request_->response_encoder_.runHighWatermarkCallbacks(); } } void ServerConnectionImpl::onBelowLowWatermark() { - if (active_request_.has_value()) { - active_request_.value().response_encoder_.runLowWatermarkCallbacks(); + if (active_request_) { + active_request_->response_encoder_.runLowWatermarkCallbacks(); } } diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index 0038f60c94cb9..c0904991cf3bf 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -467,10 +467,11 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { /** * An active HTTP/1.1 request. */ - struct ActiveRequest { + struct ActiveRequest : public Event::DeferredDeletable { ActiveRequest(ServerConnectionImpl& connection, StreamInfo::BytesMeterSharedPtr&& bytes_meter) : response_encoder_(connection, std::move(bytes_meter), connection.codec_settings_.stream_error_on_invalid_http_message_) {} + ~ActiveRequest() override = default; void dumpState(std::ostream& os, int indent_level) const; HeaderString request_url_; @@ -478,7 +479,7 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { ResponseEncoderImpl response_encoder_; bool remote_complete_{}; }; - absl::optional& activeRequest() { return active_request_; } + ActiveRequest* activeRequest() { return active_request_.get(); } // ConnectionImpl ParserStatus onMessageCompleteBase() override; // Add the size of the request_url to the reported header size when processing request headers. @@ -501,7 +502,7 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { // ConnectionImpl void onEncodeComplete() override; StreamInfo::BytesMeter& getBytesMeter() override { - if (active_request_.has_value()) { + if (active_request_) { return *(active_request_->response_encoder_.getStream().bytesMeter()); } if (bytes_meter_before_stream_ == nullptr) { @@ -550,7 +551,7 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { Status checkHeaderNameForUnderscores() override; ServerConnectionCallbacks& callbacks_; - absl::optional active_request_; + std::unique_ptr active_request_; const Buffer::OwnedBufferFragmentImpl::Releasor response_buffer_releasor_; uint32_t outbound_responses_{}; // This defaults to 2, which functionally disables pipelining. If any users diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index 1fa32ed7de37e..cd456d0bb5888 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -683,6 +683,10 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi dynamic_cast(*client).goAway(); dynamic_cast(*server).goAway(); } + + // Run deletion as would happen on the dispatchers to avoid inversion of + // lifetimes of dispatcher and connection. + server_connection.dispatcher_.to_delete_.clear(); } } // namespace diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 51a197b030cfd..2b1bdf8d0f116 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -86,6 +86,12 @@ class Http1ServerConnectionImplTest : public Http1CodecTestBase { max_request_headers_count_, headers_with_underscores_action_); } + ~Http1ServerConnectionImplTest() override { + // Run deletion as would happen on the dispatchers to avoid inversion of + // lifetimes of dispatcher and connection. + connection_.dispatcher_.to_delete_.clear(); + } + NiceMock connection_; NiceMock callbacks_; NiceMock codec_settings_; diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 0e976a30e9d9c..38558114b6845 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -308,17 +308,17 @@ class TestHttp1ServerConnectionImpl : public Http::Http1::ServerConnectionImpl { Http::Http1::ParserStatus onMessageCompleteBase() override { auto rc = ServerConnectionImpl::onMessageCompleteBase(); - if (activeRequest().has_value() && activeRequest().value().request_decoder_) { + if (activeRequest() && activeRequest()->request_decoder_) { // Undo the read disable from the base class - we have many tests which // waitForDisconnect after a full request has been read which will not // receive the disconnect if reading is disabled. - activeRequest().value().response_encoder_.readDisable(false); + activeRequest()->response_encoder_.readDisable(false); } return rc; } ~TestHttp1ServerConnectionImpl() override { - if (activeRequest().has_value()) { - activeRequest().value().response_encoder_.clearReadDisableCallsForTests(); + if (activeRequest()) { + activeRequest()->response_encoder_.clearReadDisableCallsForTests(); } } }; From dc2ac22fd004586fa47cbd6a4b096f95834e6125 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Fri, 19 Nov 2021 17:59:05 -0500 Subject: [PATCH 04/51] stream_info: reworking upstream data (#19020) Signed-off-by: Alyssa Wilk --- envoy/stream_info/stream_info.h | 105 ++++++++++- source/common/router/upstream_request.cc | 10 +- source/common/stream_info/stream_info_impl.h | 174 ++++++++++++++---- .../common/access_log/access_log_impl_test.cc | 8 +- .../stream_info/stream_info_impl_test.cc | 31 ++++ test/fuzz/utility.h | 2 +- test/mocks/stream_info/mocks.cc | 2 +- test/mocks/stream_info/mocks.h | 5 +- 8 files changed, 293 insertions(+), 44 deletions(-) diff --git a/envoy/stream_info/stream_info.h b/envoy/stream_info/stream_info.h index 28374144eff0c..9836510f861a6 100644 --- a/envoy/stream_info/stream_info.h +++ b/envoy/stream_info/stream_info.h @@ -320,6 +320,99 @@ struct BytesMeter { using BytesMeterSharedPtr = std::shared_ptr; +// TODO(alyssawilk) after landing this, remove all the duplicate getters and +// setters from StreamInfo. +class UpstreamInfo { +public: + virtual ~UpstreamInfo() = default; + + /** + * Dump the upstream info to the specified ostream. + * + * @param os the ostream to dump state to + * @param indent_level the depth, for pretty-printing. + * + * This function is called on Envoy fatal errors so should avoid memory allocation. + */ + virtual void dumpState(std::ostream& os, int indent_level = 0) const PURE; + + /** + * @param connection ID of the upstream connection. + */ + virtual void setUpstreamConnectionId(uint64_t id) PURE; + + /** + * @return the ID of the upstream connection, or absl::nullopt if not available. + */ + virtual absl::optional upstreamConnectionId() const PURE; + + /** + * @param connection_info sets the upstream ssl connection. + */ + virtual void + setUpstreamSslConnection(const Ssl::ConnectionInfoConstSharedPtr& ssl_connection_info) PURE; + + /** + * @return the upstream SSL connection. This will be nullptr if the upstream + * connection does not use SSL. + */ + virtual Ssl::ConnectionInfoConstSharedPtr upstreamSslConnection() const PURE; + + /** + * Sets the upstream timing information for this stream. This is useful for + * when multiple upstream requests are issued and we want to save timing + * information for the one that "wins". + */ + virtual void setUpstreamTiming(const UpstreamTiming& upstream_timing) PURE; + + /* + * @return the upstream timing for this stream + * */ + virtual UpstreamTiming& upstreamTiming() PURE; + virtual const UpstreamTiming& upstreamTiming() const PURE; + + /** + * @param upstream_local_address sets the local address of the upstream connection. Note that it + * can be different than the local address of the downstream connection. + */ + virtual void setUpstreamLocalAddress( + const Network::Address::InstanceConstSharedPtr& upstream_local_address) PURE; + + /** + * @return the upstream local address. + */ + virtual const Network::Address::InstanceConstSharedPtr& upstreamLocalAddress() const PURE; + + /** + * @param failure_reason the upstream transport failure reason. + */ + virtual void setUpstreamTransportFailureReason(absl::string_view failure_reason) PURE; + + /** + * @return const std::string& the upstream transport failure reason, e.g. certificate validation + * failed. + */ + virtual const std::string& upstreamTransportFailureReason() const PURE; + + /** + * @param host the selected upstream host for the request. + */ + virtual void setUpstreamHost(Upstream::HostDescriptionConstSharedPtr host) PURE; + + /** + * @return upstream host description. + */ + virtual Upstream::HostDescriptionConstSharedPtr upstreamHost() const PURE; + + /** + * Filter State object to be shared between upstream and downstream filters. + * @param pointer to upstream connections filter state. + * @return pointer to filter state to be used by upstream connections. + */ + virtual const FilterStateSharedPtr& upstreamFilterState() const PURE; + virtual void setUpstreamFilterState(const FilterStateSharedPtr& filter_state) PURE; +}; + /** * Additional information about a completed request for logging. */ @@ -432,13 +525,23 @@ class StreamInfo { */ virtual void setUpstreamTiming(const UpstreamTiming& upstream_timing) PURE; + /** + * Sets the upstream information for this stream. + */ + virtual void setUpstreamInfo(std::shared_ptr) PURE; + + /** + * Returns the upstream information for this stream. + */ + virtual std::shared_ptr upstreamInfo() PURE; + virtual OptRef upstreamInfo() const PURE; + /** * Returns the upstream timing information for this stream. * It is not expected that the fields in upstreamTiming() will be set until * the upstream request is complete. */ virtual UpstreamTiming& upstreamTiming() PURE; - virtual const UpstreamTiming& upstreamTiming() const PURE; /** * @return the duration between the first byte of the request was sent upstream and the start of diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index 3098174538746..213e71cb58a1f 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -427,10 +427,12 @@ void UpstreamRequest::onPoolReady( stream_info_.protocol(protocol.value()); } - upstream_timing_.upstream_connect_start_ = info.upstreamTiming().upstream_connect_start_; - upstream_timing_.upstream_connect_complete_ = info.upstreamTiming().upstream_connect_complete_; - upstream_timing_.upstream_handshake_complete_ = - info.upstreamTiming().upstream_handshake_complete_; + if (info.upstreamInfo().has_value()) { + auto& upstream_timing = info.upstreamInfo().value().get().upstreamTiming(); + upstream_timing_.upstream_connect_start_ = upstream_timing.upstream_connect_start_; + upstream_timing_.upstream_connect_complete_ = upstream_timing.upstream_connect_complete_; + upstream_timing_.upstream_handshake_complete_ = upstream_timing.upstream_handshake_complete_; + } stream_info_.setUpstreamFilterState(std::make_shared( info.filterState().parent()->parent(), StreamInfo::FilterState::LifeSpan::Request)); diff --git a/source/common/stream_info/stream_info_impl.h b/source/common/stream_info/stream_info_impl.h index f66f5e891de5f..bbf1bef50e4b8 100644 --- a/source/common/stream_info/stream_info_impl.h +++ b/source/common/stream_info/stream_info_impl.h @@ -34,6 +34,64 @@ const ReplacementMap& emptySpaceReplacement() { } // namespace +struct UpstreamInfoImpl : public UpstreamInfo { + void setUpstreamConnectionId(uint64_t id) override { upstream_connection_id_ = id; } + + absl::optional upstreamConnectionId() const override { return upstream_connection_id_; } + + void + setUpstreamSslConnection(const Ssl::ConnectionInfoConstSharedPtr& ssl_connection_info) override { + upstream_ssl_info_ = ssl_connection_info; + } + + Ssl::ConnectionInfoConstSharedPtr upstreamSslConnection() const override { + return upstream_ssl_info_; + } + void setUpstreamTiming(const UpstreamTiming& upstream_timing) override { + upstream_timing_ = upstream_timing; + } + UpstreamTiming& upstreamTiming() override { return upstream_timing_; } + const UpstreamTiming& upstreamTiming() const override { return upstream_timing_; } + const Network::Address::InstanceConstSharedPtr& upstreamLocalAddress() const override { + return upstream_local_address_; + } + void setUpstreamLocalAddress( + const Network::Address::InstanceConstSharedPtr& upstream_local_address) override { + upstream_local_address_ = upstream_local_address; + } + void setUpstreamTransportFailureReason(absl::string_view failure_reason) override { + upstream_transport_failure_reason_ = std::string(failure_reason); + } + const std::string& upstreamTransportFailureReason() const override { + return upstream_transport_failure_reason_; + } + void setUpstreamHost(Upstream::HostDescriptionConstSharedPtr host) override { + upstream_host_ = host; + } + const FilterStateSharedPtr& upstreamFilterState() const override { + return upstream_filter_state_; + } + void setUpstreamFilterState(const FilterStateSharedPtr& filter_state) override { + upstream_filter_state_ = filter_state; + } + + Upstream::HostDescriptionConstSharedPtr upstreamHost() const override { return upstream_host_; } + + void dumpState(std::ostream& os, int indent_level = 0) const override { + const char* spaces = spacesForLevel(indent_level); + os << spaces << "UpstreamInfoImpl " << this << DUMP_OPTIONAL_MEMBER(upstream_connection_id_) + << "\n"; + } + + Upstream::HostDescriptionConstSharedPtr upstream_host_{}; + Network::Address::InstanceConstSharedPtr upstream_local_address_; + UpstreamTiming upstream_timing_; + Ssl::ConnectionInfoConstSharedPtr upstream_ssl_info_; + absl::optional upstream_connection_id_; + std::string upstream_transport_failure_reason_; + FilterStateSharedPtr upstream_filter_state_; +}; + struct StreamInfoImpl : public StreamInfo { StreamInfoImpl( TimeSource& time_source, @@ -71,9 +129,23 @@ struct StreamInfoImpl : public StreamInfo { start_time_monotonic_); } - void setUpstreamConnectionId(uint64_t id) override { upstream_connection_id_ = id; } + void maybeCreateUpstreamInfo() { + if (!upstream_info_) { + upstream_info_ = std::make_shared(); + } + } - absl::optional upstreamConnectionId() const override { return upstream_connection_id_; } + void setUpstreamConnectionId(uint64_t id) override { + maybeCreateUpstreamInfo(); + upstream_info_->setUpstreamConnectionId(id); + } + + absl::optional upstreamConnectionId() const override { + if (!upstream_info_) { + return absl::nullopt; + } + return upstream_info_->upstreamConnectionId(); + } absl::optional lastDownstreamRxByteReceived() const override { if (!downstream_timing_.has_value()) { @@ -83,26 +155,49 @@ struct StreamInfoImpl : public StreamInfo { } void setUpstreamTiming(const UpstreamTiming& upstream_timing) override { - upstream_timing_ = upstream_timing; + maybeCreateUpstreamInfo(); + upstream_info_->setUpstreamTiming(upstream_timing); } - UpstreamTiming& upstreamTiming() override { return upstream_timing_; } - const UpstreamTiming& upstreamTiming() const override { return upstream_timing_; } + void setUpstreamInfo(std::shared_ptr info) override { upstream_info_ = info; } + std::shared_ptr upstreamInfo() override { return upstream_info_; } + OptRef upstreamInfo() const override { + if (!upstream_info_) { + return {}; + } + return *upstream_info_; + } + UpstreamTiming& upstreamTiming() override { + maybeCreateUpstreamInfo(); + return upstream_info_->upstreamTiming(); + } absl::optional firstUpstreamTxByteSent() const override { - return duration(upstream_timing_.first_upstream_tx_byte_sent_); + if (!upstream_info_) { + return absl::nullopt; + } + return duration(upstream_info_->upstreamTiming().first_upstream_tx_byte_sent_); } absl::optional lastUpstreamTxByteSent() const override { - return duration(upstream_timing_.last_upstream_tx_byte_sent_); + if (!upstream_info_) { + return absl::nullopt; + } + return duration(upstream_info_->upstreamTiming().last_upstream_tx_byte_sent_); } absl::optional firstUpstreamRxByteReceived() const override { - return duration(upstream_timing_.first_upstream_rx_byte_received_); + if (!upstream_info_) { + return absl::nullopt; + } + return duration(upstream_info_->upstreamTiming().first_upstream_rx_byte_received_); } absl::optional lastUpstreamRxByteReceived() const override { - return duration(upstream_timing_.last_upstream_rx_byte_received_); + if (!upstream_info_) { + return absl::nullopt; + } + return duration(upstream_info_->upstreamTiming().last_upstream_rx_byte_received_); } absl::optional firstDownstreamTxByteSent() const override { @@ -180,10 +275,16 @@ struct StreamInfoImpl : public StreamInfo { uint64_t responseFlags() const override { return response_flags_; } void onUpstreamHostSelected(Upstream::HostDescriptionConstSharedPtr host) override { - upstream_host_ = host; + maybeCreateUpstreamInfo(); + upstream_info_->setUpstreamHost(host); } - Upstream::HostDescriptionConstSharedPtr upstreamHost() const override { return upstream_host_; } + Upstream::HostDescriptionConstSharedPtr upstreamHost() const override { + if (!upstream_info_) { + return nullptr; + } + return upstream_info_->upstreamHost(); + } void setRouteName(absl::string_view route_name) override { route_name_ = std::string(route_name); @@ -193,11 +294,15 @@ struct StreamInfoImpl : public StreamInfo { void setUpstreamLocalAddress( const Network::Address::InstanceConstSharedPtr& upstream_local_address) override { - upstream_local_address_ = upstream_local_address; + maybeCreateUpstreamInfo(); + upstream_info_->setUpstreamLocalAddress(upstream_local_address); } const Network::Address::InstanceConstSharedPtr& upstreamLocalAddress() const override { - return upstream_local_address_; + if (!upstream_info_) { + return legacy_upstream_local_address_; + } + return upstream_info_->upstreamLocalAddress(); } bool healthCheck() const override { return health_check_request_; } @@ -209,11 +314,12 @@ struct StreamInfoImpl : public StreamInfo { } void setUpstreamSslConnection(const Ssl::ConnectionInfoConstSharedPtr& connection_info) override { - upstream_ssl_info_ = connection_info; + maybeCreateUpstreamInfo(); + upstream_info_->setUpstreamSslConnection(connection_info); } Ssl::ConnectionInfoConstSharedPtr upstreamSslConnection() const override { - return upstream_ssl_info_; + return upstream_info_ ? upstream_info_->upstreamSslConnection() : nullptr; } Router::RouteConstSharedPtr route() const override { return route_; } @@ -229,18 +335,26 @@ struct StreamInfoImpl : public StreamInfo { const FilterState& filterState() const override { return *filter_state_; } const FilterStateSharedPtr& upstreamFilterState() const override { - return upstream_filter_state_; + if (!upstream_info_) { + return legacy_upstream_filter_state_; + } + return upstream_info_->upstreamFilterState(); } void setUpstreamFilterState(const FilterStateSharedPtr& filter_state) override { - upstream_filter_state_ = filter_state; + maybeCreateUpstreamInfo(); + return upstream_info_->setUpstreamFilterState(filter_state); } void setUpstreamTransportFailureReason(absl::string_view failure_reason) override { - upstream_transport_failure_reason_ = std::string(failure_reason); + maybeCreateUpstreamInfo(); + upstream_info_->setUpstreamTransportFailureReason(failure_reason); } const std::string& upstreamTransportFailureReason() const override { - return upstream_transport_failure_reason_; + if (!upstream_info_) { + return legacy_upstream_transport_failure_reason_; + } + return upstream_info_->upstreamTransportFailureReason(); } void setRequestHeaders(const Http::RequestHeaderMap& headers) override { @@ -262,10 +376,11 @@ struct StreamInfoImpl : public StreamInfo { void dumpState(std::ostream& os, int indent_level = 0) const { const char* spaces = spacesForLevel(indent_level); - os << spaces << "StreamInfoImpl " << this << DUMP_OPTIONAL_MEMBER(upstream_connection_id_) - << DUMP_OPTIONAL_MEMBER(protocol_) << DUMP_OPTIONAL_MEMBER(response_code_) - << DUMP_OPTIONAL_MEMBER(response_code_details_) << DUMP_OPTIONAL_MEMBER(attempt_count_) - << DUMP_MEMBER(health_check_request_) << DUMP_MEMBER(route_name_) << "\n"; + os << spaces << "StreamInfoImpl " << this << DUMP_OPTIONAL_MEMBER(protocol_) + << DUMP_OPTIONAL_MEMBER(response_code_) << DUMP_OPTIONAL_MEMBER(response_code_details_) + << DUMP_OPTIONAL_MEMBER(attempt_count_) << DUMP_MEMBER(health_check_request_) + << DUMP_MEMBER(route_name_); + DUMP_DETAILS(upstream_info_); } void setUpstreamClusterInfo( @@ -282,7 +397,6 @@ struct StreamInfoImpl : public StreamInfo { } const std::string& filterChainName() const override { return filter_chain_name_; } - void setAttemptCount(uint32_t attempt_count) override { attempt_count_ = attempt_count; } absl::optional attemptCount() const override { return attempt_count_; } @@ -301,7 +415,6 @@ struct StreamInfoImpl : public StreamInfo { upstream_bytes_meter->addWireBytesReceived(upstream_bytes_meter_->wireBytesReceived()); upstream_bytes_meter->addHeaderBytesSent(upstream_bytes_meter_->headerBytesSent()); upstream_bytes_meter->addHeaderBytesReceived(upstream_bytes_meter_->headerBytesReceived()); - upstream_bytes_meter_ = upstream_bytes_meter; } @@ -323,14 +436,12 @@ struct StreamInfoImpl : public StreamInfo { absl::optional response_code_details_; absl::optional connection_termination_details_; uint64_t response_flags_{}; - Upstream::HostDescriptionConstSharedPtr upstream_host_{}; bool health_check_request_{}; Router::RouteConstSharedPtr route_; envoy::config::core::v3::Metadata metadata_{}; FilterStateSharedPtr filter_state_; - FilterStateSharedPtr upstream_filter_state_; + FilterStateSharedPtr legacy_upstream_filter_state_; std::string route_name_; - absl::optional upstream_connection_id_; absl::optional attempt_count_; private: @@ -352,17 +463,16 @@ struct StreamInfoImpl : public StreamInfo { : emptyDownstreamAddressProvider()), trace_reason_(Tracing::Reason::NotTraceable) {} + std::shared_ptr upstream_info_; uint64_t bytes_received_{}; uint64_t bytes_sent_{}; - Network::Address::InstanceConstSharedPtr upstream_local_address_; + Network::Address::InstanceConstSharedPtr legacy_upstream_local_address_; const Network::ConnectionInfoProviderSharedPtr downstream_connection_info_provider_; - Ssl::ConnectionInfoConstSharedPtr upstream_ssl_info_; std::string requested_server_name_; const Http::RequestHeaderMap* request_headers_{}; Http::RequestIdStreamInfoProviderSharedPtr request_id_provider_; absl::optional downstream_timing_; - UpstreamTiming upstream_timing_; - std::string upstream_transport_failure_reason_; + std::string legacy_upstream_transport_failure_reason_; absl::optional upstream_cluster_info_; std::string filter_chain_name_; Tracing::Reason trace_reason_; diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index 93cf53e0baf5d..d2d4c291f110d 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -106,8 +106,8 @@ name: accesslog EXPECT_CALL(*file_, write(_)); auto cluster = std::make_shared>(); - stream_info_.upstream_host_ = - Upstream::makeTestHostDescription(cluster, "tcp://10.0.0.5:1234", simTime()); + stream_info_.onUpstreamHostSelected( + Upstream::makeTestHostDescription(cluster, "tcp://10.0.0.5:1234", simTime())); stream_info_.setResponseFlag(StreamInfo::ResponseFlag::DownstreamConnectionTermination); log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); @@ -217,8 +217,8 @@ name: accesslog TEST_F(AccessLogImplTest, UpstreamHost) { auto cluster = std::make_shared>(); - stream_info_.upstream_host_ = - Upstream::makeTestHostDescription(cluster, "tcp://10.0.0.5:1234", simTime()); + stream_info_.onUpstreamHostSelected( + Upstream::makeTestHostDescription(cluster, "tcp://10.0.0.5:1234", simTime())); const std::string yaml = R"EOF( name: accesslog diff --git a/test/common/stream_info/stream_info_impl_test.cc b/test/common/stream_info/stream_info_impl_test.cc index 40343f1c04224..74ad2c1dcbabe 100644 --- a/test/common/stream_info/stream_info_impl_test.cc +++ b/test/common/stream_info/stream_info_impl_test.cc @@ -137,7 +137,9 @@ TEST_F(StreamInfoImplTest, MiscSettersAndGetters) { { StreamInfoImpl stream_info(Http::Protocol::Http2, test_time_.timeSystem(), nullptr); + EXPECT_EQ(nullptr, stream_info.upstreamInfo()); EXPECT_EQ(Http::Protocol::Http2, stream_info.protocol().value()); + EXPECT_FALSE(stream_info.upstreamConnectionId().has_value()); stream_info.protocol(Http::Protocol::Http10); EXPECT_EQ(Http::Protocol::Http10, stream_info.protocol().value()); @@ -147,6 +149,11 @@ TEST_F(StreamInfoImplTest, MiscSettersAndGetters) { ASSERT_TRUE(stream_info.responseCode()); EXPECT_EQ(200, stream_info.responseCode().value()); + EXPECT_FALSE(stream_info.attemptCount().has_value()); + stream_info.setAttemptCount(93); + ASSERT_TRUE(stream_info.attemptCount().has_value()); + EXPECT_EQ(stream_info.attemptCount().value(), 93); + EXPECT_FALSE(stream_info.responseCodeDetails().has_value()); stream_info.setResponseCodeDetails(ResponseCodeDetails::get().ViaUpstream); ASSERT_TRUE(stream_info.responseCodeDetails().has_value()); @@ -177,6 +184,7 @@ TEST_F(StreamInfoImplTest, MiscSettersAndGetters) { FilterState::LifeSpan::FilterChain); EXPECT_EQ(1, stream_info.filterState()->getDataReadOnly("test").access()); + EXPECT_EQ(nullptr, stream_info.upstreamFilterState()); stream_info.setUpstreamFilterState(stream_info.filterState()); EXPECT_EQ(1, stream_info.upstreamFilterState()->getDataReadOnly("test").access()); @@ -198,6 +206,11 @@ TEST_F(StreamInfoImplTest, MiscSettersAndGetters) { stream_info.setUpstreamConnectionId(12345); ASSERT_TRUE(stream_info.upstreamConnectionId().has_value()); EXPECT_EQ(12345, stream_info.upstreamConnectionId().value()); + + std::shared_ptr new_info = std::make_shared(); + EXPECT_NE(stream_info.upstreamInfo(), new_info); + stream_info.setUpstreamInfo(new_info); + EXPECT_EQ(stream_info.upstreamInfo(), new_info); } } @@ -266,6 +279,24 @@ TEST_F(StreamInfoImplTest, Details) { EXPECT_EQ(stream_info.responseCodeDetails().value(), "two_words"); } +TEST(UpstreamInfoImplTest, DumpState) { + UpstreamInfoImpl upstream_info; + + { + std::stringstream out; + upstream_info.dumpState(out, 0); + std::string state = out.str(); + EXPECT_THAT(state, testing::HasSubstr("upstream_connection_id_: null")); + } + upstream_info.setUpstreamConnectionId(5); + { + std::stringstream out; + upstream_info.dumpState(out, 0); + std::string state = out.str(); + EXPECT_THAT(state, testing::HasSubstr("upstream_connection_id_: 5")); + } +} + } // namespace } // namespace StreamInfo } // namespace Envoy diff --git a/test/fuzz/utility.h b/test/fuzz/utility.h index 1fc315ab732fe..6647c6cf4d591 100644 --- a/test/fuzz/utility.h +++ b/test/fuzz/utility.h @@ -157,7 +157,7 @@ inline std::unique_ptr fromStreamInfo(const test::fuzz::StreamIn auto upstream_metadata = std::make_shared( replaceInvalidStringValues(stream_info.upstream_metadata())); ON_CALL(*upstream_host, metadata()).WillByDefault(testing::Return(upstream_metadata)); - test_stream_info->upstream_host_ = upstream_host; + test_stream_info->onUpstreamHostSelected(upstream_host); auto address = stream_info.has_address() ? Envoy::Network::Address::resolveProtoAddress(stream_info.address()) : Network::Utility::resolveUrl("tcp://10.0.0.1:443"); diff --git a/test/mocks/stream_info/mocks.cc b/test/mocks/stream_info/mocks.cc index 4bdd1eaba8632..3bc5473c55385 100644 --- a/test/mocks/stream_info/mocks.cc +++ b/test/mocks/stream_info/mocks.cc @@ -142,7 +142,7 @@ MockStreamInfo::MockStreamInfo() downstream_bytes_meter_ = downstream_bytes_meter; })); ON_CALL(*this, upstreamTiming()).WillByDefault(ReturnRef(upstream_timing_)); - ON_CALL(Const(*this), upstreamTiming()).WillByDefault(ReturnRef(upstream_timing_)); + ON_CALL(Const(*this), upstreamTiming()).WillByDefault(Return(upstream_timing_)); } MockStreamInfo::~MockStreamInfo() = default; diff --git a/test/mocks/stream_info/mocks.h b/test/mocks/stream_info/mocks.h index 115a0f283c39d..4f96a1d16ff4b 100644 --- a/test/mocks/stream_info/mocks.h +++ b/test/mocks/stream_info/mocks.h @@ -31,8 +31,11 @@ class MockStreamInfo : public StreamInfo { MOCK_METHOD(MonotonicTime, startTimeMonotonic, (), (const)); MOCK_METHOD(absl::optional, lastDownstreamRxByteReceived, (), (const)); MOCK_METHOD(void, setUpstreamTiming, (const UpstreamTiming&)); + MOCK_METHOD(void, setUpstreamInfo, (std::shared_ptr)); + MOCK_METHOD(std::shared_ptr, upstreamInfo, ()); + MOCK_METHOD(OptRef, upstreamInfo, (), (const)); MOCK_METHOD(UpstreamTiming&, upstreamTiming, ()); - MOCK_METHOD(const UpstreamTiming&, upstreamTiming, (), (const)); + MOCK_METHOD(OptRef, upstreamTiming, (), (const)); MOCK_METHOD(absl::optional, firstUpstreamTxByteSent, (), (const)); MOCK_METHOD(void, onFirstUpstreamTxByteSent, ()); MOCK_METHOD(absl::optional, lastUpstreamTxByteSent, (), (const)); From 23e5fc2939e1782416155509edfca323150e8a59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20M=C4=85ka?= <62388446+michalmaka@users.noreply.github.com> Date: Sat, 20 Nov 2021 00:00:33 +0100 Subject: [PATCH 05/51] udp_proxy: added per packet load balancing possibility (#18605) Signed-off-by: Michal Maka --- .../filters/udp/udp_proxy/v3/udp_proxy.proto | 7 +- .../listeners/udp_filters/udp_proxy.rst | 15 +- docs/root/version_history/current.rst | 1 + .../filters/udp/udp_proxy/udp_proxy_filter.cc | 152 +++++++++++----- .../filters/udp/udp_proxy/udp_proxy_filter.h | 92 ++++++++-- .../udp/udp_proxy/udp_proxy_filter_test.cc | 171 ++++++++++++++++++ tools/spelling/spelling_dictionary.txt | 1 + 7 files changed, 377 insertions(+), 62 deletions(-) diff --git a/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto b/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto index 9d410e28afe3d..81d2bf54d4942 100644 --- a/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto +++ b/api/envoy/extensions/filters/udp/udp_proxy/v3/udp_proxy.proto @@ -20,7 +20,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#extension: envoy.filters.udp_listener.udp_proxy] // Configuration for the UDP proxy filter. -// [#next-free-field: 7] +// [#next-free-field: 8] message UdpProxyConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.udp.udp_proxy.v2alpha.UdpProxyConfig"; @@ -82,4 +82,9 @@ message UdpProxyConfig { // :ref:`prefer_gro ` is true for upstream // sockets as the assumption is datagrams will be received from a single source. config.core.v3.UdpSocketConfig upstream_socket_config = 6; + + // Perform per packet load balancing (upstream host selection) on each received data chunk. + // The default if not specified is false, that means each data chunk is forwarded + // to upstream host selected on first chunk receival for that "session" (identified by source IP/port and local IP/port). + bool use_per_packet_load_balancing = 7; } diff --git a/docs/root/configuration/listeners/udp_filters/udp_proxy.rst b/docs/root/configuration/listeners/udp_filters/udp_proxy.rst index 0c731c7e297ca..44250b0e0bef0 100644 --- a/docs/root/configuration/listeners/udp_filters/udp_proxy.rst +++ b/docs/root/configuration/listeners/udp_filters/udp_proxy.rst @@ -20,17 +20,26 @@ Each session is index by the 4-tuple consisting of source IP/port and local IP/p datagram is received on. Sessions last until the :ref:`idle timeout ` is reached. +Above *session stickness* could be disabled by setting :ref:`use_per_packet_load_balancing +` to true. +In that case, *per packet load balancing* is enabled. It means that upstream host is selected on every single data chunk +received by udp proxy using currently used load balancing policy. + The UDP proxy listener filter also can operate as a *transparent* proxy if the :ref:`use_original_src_ip ` -field is set. But please keep in mind that it does not forward the port to upstreams. It forwards only the IP address to upstreams. +field is set to true. But please keep in mind that it does not forward the port to upstreams. It forwards only the IP address to upstreams. Load balancing and unhealthy host handling ------------------------------------------ Envoy will fully utilize the configured load balancer for the configured upstream cluster when -load balancing UDP datagrams. When a new session is created, Envoy will associate the session +load balancing UDP datagrams. By default, when a new session is created, Envoy will associate the session with an upstream host selected using the configured load balancer. All future datagrams that -belong to the session will be routed to the same upstream host. +belong to the session will be routed to the same upstream host. However, if :ref:`use_per_packet_load_balancing +` +field is set to true, Envoy selects another upstream host on next datagram using the configured load balancer +and creates a new session if such does not exist. So in case of several upstream hosts available for the load balancer +each data chunk is forwarded to a different host. When an upstream host becomes unhealthy (due to :ref:`active health checking `), Envoy will attempt to create a new session to a healthy host diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 71f0ae1b60688..1ffd6ac11714f 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -84,6 +84,7 @@ New Features * tls_inspector filter: added :ref:`enable_ja3_fingerprinting ` to create JA3 fingerprint hash from Client Hello message. * transport_socket: added :ref:`envoy.transport_sockets.tcp_stats ` which generates additional statistics gathered from the OS TCP stack. * udp: add support for multiple listener filters. +* udp_proxy: added :ref:`use_per_packet_load_balancing ` option to enable per packet load balancing (selection of upstream host on each data chunk). * upstream: added the ability to :ref:`configure max connection duration ` for upstream clusters. * vcl_socket_interface: added VCL socket interface extension for fd.io VPP integration to :ref:`contrib images `. This can be enabled via :ref:`VCL ` configuration. * xds: re-introduced unified delta and sotw xDS multiplexers that share most of the implementation. Added a new runtime config ``envoy.reloadable_features.unified_mux`` (disabled by default) that when enabled, switches xDS to use unified multiplexers. diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc index 9be3f1003a37e..d67e52e6b4102 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc @@ -27,8 +27,13 @@ void UdpProxyFilter::onClusterAddOrUpdate(Upstream::ThreadLocalCluster& cluster) } ENVOY_LOG(debug, "udp proxy: attaching to cluster {}", cluster.info()->name()); - ASSERT(cluster_info_ == absl::nullopt || &cluster_info_.value().cluster_ != &cluster); - cluster_info_.emplace(*this, cluster); + ASSERT(cluster_info_ == absl::nullopt || &cluster_info_.value()->cluster_ != &cluster); + + if (config_->usingPerPacketLoadBalancing()) { + cluster_info_.emplace(std::make_unique(*this, cluster)); + } else { + cluster_info_.emplace(std::make_unique(*this, cluster)); + } } void UdpProxyFilter::onClusterRemoval(const std::string& cluster) { @@ -46,7 +51,7 @@ Network::FilterStatus UdpProxyFilter::onData(Network::UdpRecvData& data) { return Network::FilterStatus::StopIteration; } - return cluster_info_.value().onData(data); + return cluster_info_.value()->onData(data); } Network::FilterStatus UdpProxyFilter::onReceiveError(Api::IoError::IoErrorCode) { @@ -56,9 +61,10 @@ Network::FilterStatus UdpProxyFilter::onReceiveError(Api::IoError::IoErrorCode) } UdpProxyFilter::ClusterInfo::ClusterInfo(UdpProxyFilter& filter, - Upstream::ThreadLocalCluster& cluster) + Upstream::ThreadLocalCluster& cluster, + SessionStorageType&& sessions) : filter_(filter), cluster_(cluster), - cluster_stats_(generateStats(cluster.info()->statsScope())), + cluster_stats_(generateStats(cluster.info()->statsScope())), sessions_(std::move(sessions)), member_update_cb_handle_(cluster.prioritySet().addMemberUpdateCb( [this](const Upstream::HostVector&, const Upstream::HostVector& hosts_removed) { for (const auto& host : hosts_removed) { @@ -85,36 +91,84 @@ UdpProxyFilter::ClusterInfo::~ClusterInfo() { ASSERT(host_to_sessions_.empty()); } -Network::FilterStatus UdpProxyFilter::ClusterInfo::onData(Network::UdpRecvData& data) { +void UdpProxyFilter::ClusterInfo::removeSession(const ActiveSession* session) { + // First remove from the host to sessions map. + ASSERT(host_to_sessions_[&session->host()].count(session) == 1); + auto host_sessions_it = host_to_sessions_.find(&session->host()); + host_sessions_it->second.erase(session); + if (host_sessions_it->second.empty()) { + host_to_sessions_.erase(host_sessions_it); + } + + // Now remove it from the primary map. + ASSERT(sessions_.count(session) == 1); + sessions_.erase(session); +} + +UdpProxyFilter::ActiveSession* +UdpProxyFilter::ClusterInfo::createSession(Network::UdpRecvData::LocalPeerAddresses&& addresses, + const Upstream::HostConstSharedPtr& optional_host) { + if (!cluster_.info() + ->resourceManager(Upstream::ResourcePriority::Default) + .connections() + .canCreate()) { + ENVOY_LOG(debug, "cannot create new connection."); + cluster_.info()->stats().upstream_cx_overflow_.inc(); + return nullptr; + } + + if (optional_host) { + return createSessionWithHost(std::move(addresses), optional_host); + } + + auto host = chooseHost(addresses.peer_); + if (host == nullptr) { + ENVOY_LOG(debug, "cannot find any valid host."); + cluster_.info()->stats().upstream_cx_none_healthy_.inc(); + return nullptr; + } + return createSessionWithHost(std::move(addresses), host); +} + +UdpProxyFilter::ActiveSession* UdpProxyFilter::ClusterInfo::createSessionWithHost( + Network::UdpRecvData::LocalPeerAddresses&& addresses, + const Upstream::HostConstSharedPtr& host) { + ASSERT(host); + auto new_session = std::make_unique(*this, std::move(addresses), host); + auto new_session_ptr = new_session.get(); + sessions_.emplace(std::move(new_session)); + host_to_sessions_[host.get()].emplace(new_session_ptr); + return new_session_ptr; +} + +Upstream::HostConstSharedPtr UdpProxyFilter::ClusterInfo::chooseHost( + const Network::Address::InstanceConstSharedPtr& peer_address) const { + UdpLoadBalancerContext context(filter_.config_->hashPolicy(), peer_address); + Upstream::HostConstSharedPtr host = cluster_.loadBalancer().chooseHost(&context); + return host; +} + +UdpProxyFilter::StickySessionClusterInfo::StickySessionClusterInfo( + UdpProxyFilter& filter, Upstream::ThreadLocalCluster& cluster) + : ClusterInfo(filter, cluster, + SessionStorageType(1, HeterogeneousActiveSessionHash(false), + HeterogeneousActiveSessionEqual(false))) {} + +Network::FilterStatus UdpProxyFilter::StickySessionClusterInfo::onData(Network::UdpRecvData& data) { const auto active_session_it = sessions_.find(data.addresses_); ActiveSession* active_session; if (active_session_it == sessions_.end()) { - if (!cluster_.info() - ->resourceManager(Upstream::ResourcePriority::Default) - .connections() - .canCreate()) { - cluster_.info()->stats().upstream_cx_overflow_.inc(); + active_session = createSession(std::move(data.addresses_)); + if (active_session == nullptr) { return Network::FilterStatus::StopIteration; } - - UdpLoadBalancerContext context(filter_.config_->hashPolicy(), data.addresses_.peer_); - Upstream::HostConstSharedPtr host = cluster_.loadBalancer().chooseHost(&context); - if (host == nullptr) { - ENVOY_LOG(debug, "cannot find any valid host. failed to create a session."); - cluster_.info()->stats().upstream_cx_none_healthy_.inc(); - return Network::FilterStatus::StopIteration; - } - - active_session = createSession(std::move(data.addresses_), host); } else { active_session = active_session_it->get(); if (active_session->host().health() == Upstream::Host::Health::Unhealthy) { // If a host becomes unhealthy, we optimally would like to replace it with a new session // to a healthy host. We may eventually want to make this behavior configurable, but for now // this will be the universal behavior. - - UdpLoadBalancerContext context(filter_.config_->hashPolicy(), data.addresses_.peer_); - Upstream::HostConstSharedPtr host = cluster_.loadBalancer().chooseHost(&context); + auto host = chooseHost(data.addresses_.peer_); if (host != nullptr && host->health() != Upstream::Host::Health::Unhealthy && host.get() != &active_session->host()) { ENVOY_LOG(debug, "upstream session unhealthy, recreating the session"); @@ -132,28 +186,40 @@ Network::FilterStatus UdpProxyFilter::ClusterInfo::onData(Network::UdpRecvData& return Network::FilterStatus::StopIteration; } -UdpProxyFilter::ActiveSession* -UdpProxyFilter::ClusterInfo::createSession(Network::UdpRecvData::LocalPeerAddresses&& addresses, - const Upstream::HostConstSharedPtr& host) { - auto new_session = std::make_unique(*this, std::move(addresses), host); - auto new_session_ptr = new_session.get(); - sessions_.emplace(std::move(new_session)); - host_to_sessions_[host.get()].emplace(new_session_ptr); - return new_session_ptr; -} +UdpProxyFilter::PerPacketLoadBalancingClusterInfo::PerPacketLoadBalancingClusterInfo( + UdpProxyFilter& filter, Upstream::ThreadLocalCluster& cluster) + : ClusterInfo(filter, cluster, + SessionStorageType(1, HeterogeneousActiveSessionHash(true), + HeterogeneousActiveSessionEqual(true))) {} + +Network::FilterStatus +UdpProxyFilter::PerPacketLoadBalancingClusterInfo::onData(Network::UdpRecvData& data) { + auto host = chooseHost(data.addresses_.peer_); + if (host == nullptr) { + ENVOY_LOG(debug, "cannot find any valid host."); + cluster_.info()->stats().upstream_cx_none_healthy_.inc(); + return Network::FilterStatus::StopIteration; + } -void UdpProxyFilter::ClusterInfo::removeSession(const ActiveSession* session) { - // First remove from the host to sessions map. - ASSERT(host_to_sessions_[&session->host()].count(session) == 1); - auto host_sessions_it = host_to_sessions_.find(&session->host()); - host_sessions_it->second.erase(session); - if (host_sessions_it->second.empty()) { - host_to_sessions_.erase(host_sessions_it); + ENVOY_LOG(debug, "selected {} host as upstream.", host->address()->asStringView()); + + LocalPeerHostAddresses key{data.addresses_, *host}; + const auto active_session_it = sessions_.find(key); + ActiveSession* active_session; + if (active_session_it == sessions_.end()) { + active_session = createSession(std::move(data.addresses_), host); + if (active_session == nullptr) { + return Network::FilterStatus::StopIteration; + } + } else { + active_session = active_session_it->get(); + ENVOY_LOG(trace, "found already existing session on host {}.", + active_session->host().address()->asStringView()); } - // Now remove it from the primary map. - ASSERT(sessions_.count(session) == 1); - sessions_.erase(session); + active_session->write(*data.buffer_); + + return Network::FilterStatus::StopIteration; } UdpProxyFilter::ActiveSession::ActiveSession(ClusterInfo& cluster, diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h index 0d349fe3fd259..99c9708543e5c 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h @@ -70,6 +70,7 @@ class UdpProxyFilterConfig { : cluster_manager_(cluster_manager), time_source_(time_source), cluster_(config.cluster()), session_timeout_(PROTOBUF_GET_MS_OR_DEFAULT(config, idle_timeout, 60 * 1000)), use_original_src_ip_(config.use_original_src_ip()), + use_per_packet_load_balancing_(config.use_per_packet_load_balancing()), stats_(generateStats(config.stat_prefix(), root_scope)), // Default prefer_gro to true for upstream client traffic. upstream_socket_config_(config.upstream_socket_config(), true) { @@ -87,6 +88,7 @@ class UdpProxyFilterConfig { Upstream::ClusterManager& clusterManager() const { return cluster_manager_; } std::chrono::milliseconds sessionTimeout() const { return session_timeout_; } bool usingOriginalSrcIp() const { return use_original_src_ip_; } + bool usingPerPacketLoadBalancing() const { return use_per_packet_load_balancing_; } const Udp::HashPolicy* hashPolicy() const { return hash_policy_.get(); } UdpProxyDownstreamStats& stats() const { return stats_; } TimeSource& timeSource() const { return time_source_; } @@ -107,6 +109,7 @@ class UdpProxyFilterConfig { const std::string cluster_; const std::chrono::milliseconds session_timeout_; const bool use_original_src_ip_; + const bool use_per_packet_load_balancing_; std::unique_ptr hash_policy_; mutable UdpProxyDownstreamStats stats_; const Network::ResolvedUdpSocketConfig upstream_socket_config_; @@ -200,6 +203,11 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, using ActiveSessionPtr = std::unique_ptr; + struct LocalPeerHostAddresses { + const Network::UdpRecvData::LocalPeerAddresses& local_peer_addresses_; + const Upstream::Host& host_; + }; + struct HeterogeneousActiveSessionHash { // Specifying is_transparent indicates to the library infrastructure that // type-conversions should not be applied when calling find(), but instead @@ -210,31 +218,52 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, // using it in the context of absl. using is_transparent = void; // NOLINT(readability-identifier-naming) + HeterogeneousActiveSessionHash(const bool consider_host) : consider_host_(consider_host) {} + size_t operator()(const Network::UdpRecvData::LocalPeerAddresses& value) const { return absl::Hash()(value); } - size_t operator()(const ActiveSessionPtr& value) const { - return absl::Hash()(value->addresses()); + size_t operator()(const LocalPeerHostAddresses& value) const { + auto hash = this->operator()(value.local_peer_addresses_); + if (consider_host_) { + hash = absl::HashOf(hash, value.host_.address()->asStringView()); + } + return hash; } size_t operator()(const ActiveSession* value) const { - return absl::Hash()(value->addresses()); + LocalPeerHostAddresses key{value->addresses(), value->host()}; + return this->operator()(key); } + size_t operator()(const ActiveSessionPtr& value) const { return this->operator()(value.get()); } + + private: + const bool consider_host_; }; struct HeterogeneousActiveSessionEqual { // See description for HeterogeneousActiveSessionHash::is_transparent. using is_transparent = void; // NOLINT(readability-identifier-naming) + HeterogeneousActiveSessionEqual(const bool consider_host) : consider_host_(consider_host) {} + bool operator()(const ActiveSessionPtr& lhs, const Network::UdpRecvData::LocalPeerAddresses& rhs) const { return lhs->addresses() == rhs; } - bool operator()(const ActiveSessionPtr& lhs, const ActiveSessionPtr& rhs) const { - return lhs->addresses() == rhs->addresses(); + bool operator()(const ActiveSessionPtr& lhs, const LocalPeerHostAddresses& rhs) const { + return this->operator()(lhs, rhs.local_peer_addresses_) && + (consider_host_ ? &lhs->host() == &rhs.host_ : true); } bool operator()(const ActiveSessionPtr& lhs, const ActiveSession* rhs) const { - return lhs->addresses() == rhs->addresses(); + LocalPeerHostAddresses key{rhs->addresses(), rhs->host()}; + return this->operator()(lhs, key); } + bool operator()(const ActiveSessionPtr& lhs, const ActiveSessionPtr& rhs) const { + return this->operator()(lhs, rhs.get()); + } + + private: + const bool consider_host_; }; /** @@ -242,32 +271,65 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, * we will very likely support different types of routing to multiple upstream clusters. */ class ClusterInfo { + protected: + using SessionStorageType = absl::flat_hash_set; + public: - ClusterInfo(UdpProxyFilter& filter, Upstream::ThreadLocalCluster& cluster); - ~ClusterInfo(); - Network::FilterStatus onData(Network::UdpRecvData& data); + ClusterInfo(UdpProxyFilter& filter, Upstream::ThreadLocalCluster& cluster, + SessionStorageType&& sessions); + virtual ~ClusterInfo(); + virtual Network::FilterStatus onData(Network::UdpRecvData& data) PURE; void removeSession(const ActiveSession* session); UdpProxyFilter& filter_; Upstream::ThreadLocalCluster& cluster_; UdpProxyUpstreamStats cluster_stats_; - private: + protected: ActiveSession* createSession(Network::UdpRecvData::LocalPeerAddresses&& addresses, - const Upstream::HostConstSharedPtr& host); + const Upstream::HostConstSharedPtr& optional_host = nullptr); + Upstream::HostConstSharedPtr + chooseHost(const Network::Address::InstanceConstSharedPtr& peer_address) const; + + SessionStorageType sessions_; + + private: static UdpProxyUpstreamStats generateStats(Stats::Scope& scope) { const auto final_prefix = "udp"; return {ALL_UDP_PROXY_UPSTREAM_STATS(POOL_COUNTER_PREFIX(scope, final_prefix))}; } + ActiveSession* createSessionWithHost(Network::UdpRecvData::LocalPeerAddresses&& addresses, + const Upstream::HostConstSharedPtr& host); Envoy::Common::CallbackHandlePtr member_update_cb_handle_; - absl::flat_hash_set - sessions_; absl::flat_hash_map> host_to_sessions_; }; + using ClusterInfoPtr = std::unique_ptr; + + /** + * Performs forwarding and replying data to one upstream host, selected when the first datagram + * for a session is received. If the upstream host becomes unhealthy, a new one is selected. + */ + class StickySessionClusterInfo : public ClusterInfo { + public: + StickySessionClusterInfo(UdpProxyFilter& filter, Upstream::ThreadLocalCluster& cluster); + Network::FilterStatus onData(Network::UdpRecvData& data) override; + }; + + /** + * On each data chunk selects another host using underlying load balancing method and communicates + * with that host. + */ + class PerPacketLoadBalancingClusterInfo : public ClusterInfo { + public: + PerPacketLoadBalancingClusterInfo(UdpProxyFilter& filter, + Upstream::ThreadLocalCluster& cluster); + Network::FilterStatus onData(Network::UdpRecvData& data) override; + }; + virtual Network::SocketPtr createSocket(const Upstream::HostConstSharedPtr& host) { // Virtual so this can be overridden in unit tests. return std::make_unique(Network::Socket::Type::Datagram, host->address(), @@ -283,7 +345,7 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, // Right now we support a single cluster to route to. It is highly likely in the future that // we will support additional routing options either using filter chain matching, weighting, // etc. - absl::optional cluster_info_; + absl::optional cluster_info_; }; } // namespace UdpProxy diff --git a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc index f59a51f7a833c..60a01885b00fb 100644 --- a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc +++ b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc @@ -21,6 +21,7 @@ using testing::AtLeast; using testing::ByMove; using testing::DoAll; +using testing::DoDefault; using testing::InSequence; using testing::InvokeWithoutArgs; using testing::Return; @@ -639,6 +640,176 @@ TEST_F(UdpProxyFilterTest, SocketOptionForUseOriginalSrcIp) { ensureIpTransparentSocketOptions(upstream_address_, "10.0.0.2:80", 1, 0); } +// Verify that on second data packet sent from the client, another upstream host is selected. +TEST_F(UdpProxyFilterTest, PerPacketLoadBalancingBasicFlow) { + InSequence s; + + setup(R"EOF( +stat_prefix: foo +cluster: fake_cluster +use_per_packet_load_balancing: true + )EOF"); + + // Allow for two sessions. + cluster_manager_.thread_local_cluster_.cluster_.info_->resetResourceManager(2, 0, 0, 0, 0); + + expectSessionCreate(upstream_address_); + test_sessions_[0].expectWriteToUpstream("hello"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); + checkTransferStats(5 /*rx_bytes*/, 1 /*rx_datagrams*/, 0 /*tx_bytes*/, 0 /*tx_datagrams*/); + test_sessions_[0].recvDataFromUpstream("world"); + checkTransferStats(5 /*rx_bytes*/, 1 /*rx_datagrams*/, 5 /*tx_bytes*/, 1 /*tx_datagrams*/); + + auto new_host_address = Network::Utility::parseInternetAddressAndPort("20.0.0.2:443"); + auto new_host = createHost(new_host_address); + EXPECT_CALL(cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)).WillOnce(Return(new_host)); + expectSessionCreate(new_host_address); + test_sessions_[1].expectWriteToUpstream("hello2"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello2"); + EXPECT_EQ(2, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(2, config_->stats().downstream_sess_active_.value()); + checkTransferStats(11 /*rx_bytes*/, 2 /*rx_datagrams*/, 5 /*tx_bytes*/, 1 /*tx_datagrams*/); + + // On next datagram, first session should be used + EXPECT_CALL(cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)) + .WillRepeatedly(DoDefault()); + test_sessions_[0].expectWriteToUpstream("hello3"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello3"); + EXPECT_EQ(2, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(2, config_->stats().downstream_sess_active_.value()); + checkTransferStats(17 /*rx_bytes*/, 3 /*rx_datagrams*/, 5 /*tx_bytes*/, 1 /*tx_datagrams*/); +} + +// Verify that when no host is available, message is dropped. +TEST_F(UdpProxyFilterTest, PerPacketLoadBalancingFirstInvalidHost) { + InSequence s; + + setup(R"EOF( +stat_prefix: foo +cluster: fake_cluster +use_per_packet_load_balancing: true + )EOF"); + + EXPECT_CALL(cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)).WillOnce(Return(nullptr)); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + EXPECT_EQ(0, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(0, config_->stats().downstream_sess_active_.value()); + EXPECT_EQ(1, cluster_manager_.thread_local_cluster_.cluster_.info_->stats_ + .upstream_cx_none_healthy_.value()); +} + +// Verify that when on second packet no host is available, message is dropped. +TEST_F(UdpProxyFilterTest, PerPacketLoadBalancingSecondInvalidHost) { + InSequence s; + + setup(R"EOF( +stat_prefix: foo +cluster: fake_cluster +use_per_packet_load_balancing: true + )EOF"); + + expectSessionCreate(upstream_address_); + test_sessions_[0].expectWriteToUpstream("hello"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); + EXPECT_EQ(0, cluster_manager_.thread_local_cluster_.cluster_.info_->stats_ + .upstream_cx_none_healthy_.value()); + + EXPECT_CALL(cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)).WillOnce(Return(nullptr)); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello2"); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); + EXPECT_EQ(1, cluster_manager_.thread_local_cluster_.cluster_.info_->stats_ + .upstream_cx_none_healthy_.value()); +} + +// Verify that all sessions for a host are removed when a host is removed. +TEST_F(UdpProxyFilterTest, PerPacketLoadBalancingRemoveHostSessions) { + InSequence s; + + setup(R"EOF( +stat_prefix: foo +cluster: fake_cluster +use_per_packet_load_balancing: true + )EOF"); + + expectSessionCreate(upstream_address_); + test_sessions_[0].expectWriteToUpstream("hello"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); + + cluster_manager_.thread_local_cluster_.cluster_.priority_set_.runUpdateCallbacks( + 0, {}, {cluster_manager_.thread_local_cluster_.lb_.host_}); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(0, config_->stats().downstream_sess_active_.value()); + + expectSessionCreate(upstream_address_); + test_sessions_[1].expectWriteToUpstream("hello2"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello2"); + EXPECT_EQ(2, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); +} + +// Verify that all sessions for hosts in cluster are removed when a cluster is removed. +TEST_F(UdpProxyFilterTest, PerPacketLoadBalancingRemoveCluster) { + InSequence s; + + setup(R"EOF( +stat_prefix: foo +cluster: fake_cluster +use_per_packet_load_balancing: true + )EOF"); + + // Allow for two sessions. + cluster_manager_.thread_local_cluster_.cluster_.info_->resetResourceManager(2, 0, 0, 0, 0); + + expectSessionCreate(upstream_address_); + test_sessions_[0].expectWriteToUpstream("hello"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + EXPECT_EQ(1, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(1, config_->stats().downstream_sess_active_.value()); + + auto new_host_address = Network::Utility::parseInternetAddressAndPort("20.0.0.2:443"); + auto new_host = createHost(new_host_address); + EXPECT_CALL(cluster_manager_.thread_local_cluster_.lb_, chooseHost(_)).WillOnce(Return(new_host)); + expectSessionCreate(new_host_address); + test_sessions_[1].expectWriteToUpstream("hello2"); + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello2"); + EXPECT_EQ(2, config_->stats().downstream_sess_total_.value()); + EXPECT_EQ(2, config_->stats().downstream_sess_active_.value()); + + // Remove a cluster we don't care about. + cluster_update_callbacks_->onClusterRemoval("other_cluster"); + EXPECT_EQ(2, config_->stats().downstream_sess_active_.value()); + + // Remove the cluster we do care about. This should purge all sessions. + cluster_update_callbacks_->onClusterRemoval("fake_cluster"); + EXPECT_EQ(0, config_->stats().downstream_sess_active_.value()); +} + +// Verify that specific stat is included when connection limit is hit. +TEST_F(UdpProxyFilterTest, PerPacketLoadBalancingCannotCreateConnection) { + InSequence s; + + setup(R"EOF( +stat_prefix: foo +cluster: fake_cluster +use_per_packet_load_balancing: true + )EOF"); + + // Don't allow for any session. + cluster_manager_.thread_local_cluster_.cluster_.info_->resetResourceManager(0, 0, 0, 0, 0); + + recvDataFromDownstream("10.0.0.1:1000", "10.0.0.2:80", "hello"); + EXPECT_EQ( + 1, + cluster_manager_.thread_local_cluster_.cluster_.info_->stats_.upstream_cx_overflow_.value()); +} + // Make sure socket option is set correctly if use_original_src_ip is set in case of ipv6. TEST_F(UdpProxyFilterIpv6Test, SocketOptionForUseOriginalSrcIpInCaseOfIpv6) { if (!isTransparentSocketOptionsSupported()) { diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 20ebff34f2a4e..6df79bdbfbf64 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -1011,6 +1011,7 @@ rebalancer rebalancing rebuffer rebuilder +receival reconnection recurse recv From 0d0cf0dd939d5464f0388341f53027f308165f30 Mon Sep 17 00:00:00 2001 From: Jose Ulises Nino Rivera Date: Fri, 19 Nov 2021 15:01:45 -0800 Subject: [PATCH 06/51] os syscalls: add getifaddrs to singleton (#18821) Signed-off-by: Jose Nino --- envoy/api/BUILD | 4 ++ envoy/api/os_sys_calls.h | 44 ++++++++++++++ envoy/common/platform.h | 12 ---- envoy/network/BUILD | 3 - envoy/network/address.h | 1 - envoy/network/io_handle.h | 1 + source/common/api/BUILD | 1 + source/common/api/posix/os_sys_calls_impl.cc | 64 ++++++++++++++++++++ source/common/api/posix/os_sys_calls_impl.h | 2 + source/common/api/win32/os_sys_calls_impl.cc | 14 +++++ source/common/api/win32/os_sys_calls_impl.h | 2 + source/common/network/BUILD | 1 - source/common/network/utility.cc | 32 +++------- test/common/api/BUILD | 17 ++++++ test/common/api/os_sys_calls_test.cc | 34 +++++++++++ test/common/http/conn_pool_grid_test.cc | 16 +++++ test/mocks/api/mocks.h | 3 + tools/spelling/spelling_dictionary.txt | 2 + 18 files changed, 213 insertions(+), 40 deletions(-) create mode 100644 test/common/api/BUILD create mode 100644 test/common/api/os_sys_calls_test.cc diff --git a/envoy/api/BUILD b/envoy/api/BUILD index cbdc13440690c..3ac0872aac617 100644 --- a/envoy/api/BUILD +++ b/envoy/api/BUILD @@ -36,4 +36,8 @@ envoy_cc_library( "os_sys_calls_hot_restart.h", "os_sys_calls_linux.h", ], + external_deps = ["abseil_optional"], + deps = [ + "//envoy/network:address_interface", + ], ) diff --git a/envoy/api/os_sys_calls.h b/envoy/api/os_sys_calls.h index 19a25d3a55eb8..3c9b6f9698ac4 100644 --- a/envoy/api/os_sys_calls.h +++ b/envoy/api/os_sys_calls.h @@ -5,10 +5,14 @@ #include #include #include +#include #include "envoy/api/os_sys_calls_common.h" #include "envoy/common/platform.h" #include "envoy/common/pure.h" +#include "envoy/network/address.h" + +#include "absl/types/optional.h" namespace Envoy { namespace Api { @@ -17,6 +21,23 @@ struct EnvoyTcpInfo { std::chrono::microseconds tcpi_rtt; }; +// Small struct to avoid exposing ifaddrs -- which is not defined in all platforms -- to the +// codebase. +struct InterfaceAddress { + InterfaceAddress(absl::string_view interface_name, unsigned int interface_flags, + Envoy::Network::Address::InstanceConstSharedPtr interface_addr) + : interface_name_(interface_name), interface_flags_(interface_flags), + interface_addr_(interface_addr) {} + + std::string interface_name_; + unsigned int interface_flags_; + Envoy::Network::Address::InstanceConstSharedPtr interface_addr_; +}; + +using InterfaceAddressVector = std::vector; + +using AlternateGetifaddrs = std::function; + class OsSysCalls { public: virtual ~OsSysCalls() = default; @@ -195,6 +216,29 @@ class OsSysCalls { * @see man TCP_INFO. Get the tcp info for the socket. */ virtual SysCallBoolResult socketTcpInfo(os_fd_t sockfd, EnvoyTcpInfo* tcp_info) PURE; + + /** + * return true if the OS supports getifaddrs. + */ + virtual bool supportsGetifaddrs() const PURE; + + /** + * @see man getifaddrs + */ + virtual SysCallIntResult getifaddrs(InterfaceAddressVector& interfaces) PURE; + + /** + * allows a platform to override getifaddrs or provide an implementation if one does not exist + * natively. + * + * @arg alternate_getifaddrs function pointer to implementation. + */ + virtual void setAlternateGetifaddrs(AlternateGetifaddrs alternate_getifaddrs) { + alternate_getifaddrs_ = alternate_getifaddrs; + } + +protected: + absl::optional alternate_getifaddrs_{}; }; using OsSysCallsPtr = std::unique_ptr; diff --git a/envoy/common/platform.h b/envoy/common/platform.h index 8ae695d46c274..c56c18b0e6b69 100644 --- a/envoy/common/platform.h +++ b/envoy/common/platform.h @@ -301,18 +301,6 @@ struct mmsghdr { }; #endif -#define SUPPORTS_GETIFADDRS -#ifdef WIN32 -#undef SUPPORTS_GETIFADDRS -#endif - -// https://android.googlesource.com/platform/prebuilts/ndk/+/dev/platform/sysroot/usr/include/ifaddrs.h -#ifdef __ANDROID_API__ -#if __ANDROID_API__ < 24 -#undef SUPPORTS_GETIFADDRS -#endif // __ANDROID_API__ < 24 -#endif // ifdef __ANDROID_API__ - // TODO: Remove once bazel supports NDKs > 21 #define SUPPORTS_CPP_17_CONTIGUOUS_ITERATOR #ifdef __ANDROID_API__ diff --git a/envoy/network/BUILD b/envoy/network/BUILD index d454518fbdbf4..77c31f791aa81 100644 --- a/envoy/network/BUILD +++ b/envoy/network/BUILD @@ -11,9 +11,6 @@ envoy_package() envoy_cc_library( name = "address_interface", hdrs = ["address.h"], - deps = [ - "//envoy/api:os_sys_calls_interface", - ], ) envoy_cc_library( diff --git a/envoy/network/address.h b/envoy/network/address.h index bd28205bd6305..2b50b0728168e 100644 --- a/envoy/network/address.h +++ b/envoy/network/address.h @@ -7,7 +7,6 @@ #include #include -#include "envoy/api/os_sys_calls.h" #include "envoy/common/platform.h" #include "envoy/common/pure.h" diff --git a/envoy/network/io_handle.h b/envoy/network/io_handle.h index a960887a56a1c..12a45cf0c0d3b 100644 --- a/envoy/network/io_handle.h +++ b/envoy/network/io_handle.h @@ -4,6 +4,7 @@ #include #include "envoy/api/io_error.h" +#include "envoy/api/os_sys_calls_common.h" #include "envoy/common/platform.h" #include "envoy/common/pure.h" #include "envoy/event/file_event.h" diff --git a/source/common/api/BUILD b/source/common/api/BUILD index 07443785ab89f..7ed8334ae8823 100644 --- a/source/common/api/BUILD +++ b/source/common/api/BUILD @@ -48,6 +48,7 @@ envoy_cc_library( }), deps = [ "//envoy/api:os_sys_calls_interface", + "//source/common/network:address_lib", "//source/common/singleton:threadsafe_singleton", ], ) diff --git a/source/common/api/posix/os_sys_calls_impl.cc b/source/common/api/posix/os_sys_calls_impl.cc index d4c05f77123b5..5411580a212b8 100644 --- a/source/common/api/posix/os_sys_calls_impl.cc +++ b/source/common/api/posix/os_sys_calls_impl.cc @@ -6,6 +6,7 @@ #include #include "source/common/api/os_sys_calls_impl.h" +#include "source/common/network/address_impl.h" namespace Envoy { namespace Api { @@ -296,5 +297,68 @@ SysCallBoolResult OsSysCallsImpl::socketTcpInfo([[maybe_unused]] os_fd_t sockfd, return {false, EOPNOTSUPP}; } +bool OsSysCallsImpl::supportsGetifaddrs() const { +// TODO: eliminate this branching by upstreaming an alternative Android implementation +// e.g.: https://github.com/envoyproxy/envoy-mobile/blob/main/third_party/android/ifaddrs-android.h +#if defined(__ANDROID_API__) && __ANDROID_API__ < 24 + if (alternate_getifaddrs_.has_value()) { + return true; + } + return false; +#else + // Note: posix defaults to true regardless of whether an alternate getifaddrs has been set or not. + // This is because as far as we are aware only Android<24 lacks an implementation and thus another + // posix based platform that lacks a native getifaddrs implementation should be a programming + // error. + // + // That being said, if an alternate getifaddrs impl is set, that will be used in calls to + // OsSysCallsImpl::getifaddrs as seen below. + return true; +#endif +} + +SysCallIntResult OsSysCallsImpl::getifaddrs([[maybe_unused]] InterfaceAddressVector& interfaces) { + if (alternate_getifaddrs_.has_value()) { + return alternate_getifaddrs_.value()(interfaces); + } + +// TODO: eliminate this branching by upstreaming an alternative Android implementation +// e.g.: https://github.com/envoyproxy/envoy-mobile/blob/main/third_party/android/ifaddrs-android.h +#if defined(__ANDROID_API__) && __ANDROID_API__ < 24 + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +#else + struct ifaddrs* ifaddr; + struct ifaddrs* ifa; + + const int rc = ::getifaddrs(&ifaddr); + if (rc == -1) { + return {rc, errno}; + } + + for (ifa = ifaddr; ifa != nullptr; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == nullptr) { + continue; + } + + if (ifa->ifa_addr->sa_family == AF_INET || ifa->ifa_addr->sa_family == AF_INET6) { + const sockaddr_storage* ss = reinterpret_cast(ifa->ifa_addr); + size_t ss_len = + ifa->ifa_addr->sa_family == AF_INET ? sizeof(sockaddr_in) : sizeof(sockaddr_in6); + StatusOr address = + Network::Address::addressFromSockAddr(*ss, ss_len, ifa->ifa_addr->sa_family == AF_INET6); + if (address.ok()) { + interfaces.emplace_back(ifa->ifa_name, ifa->ifa_flags, *address); + } + } + } + + if (ifaddr) { + ::freeifaddrs(ifaddr); + } + + return {rc, 0}; +#endif +} + } // namespace Api } // namespace Envoy diff --git a/source/common/api/posix/os_sys_calls_impl.h b/source/common/api/posix/os_sys_calls_impl.h index f03bcc24bc7d9..20b9adf56ef24 100644 --- a/source/common/api/posix/os_sys_calls_impl.h +++ b/source/common/api/posix/os_sys_calls_impl.h @@ -50,6 +50,8 @@ class OsSysCallsImpl : public OsSysCalls { SysCallSocketResult duplicate(os_fd_t oldfd) override; SysCallSocketResult accept(os_fd_t socket, sockaddr* addr, socklen_t* addrlen) override; SysCallBoolResult socketTcpInfo(os_fd_t sockfd, EnvoyTcpInfo* tcp_info) override; + bool supportsGetifaddrs() const override; + SysCallIntResult getifaddrs(InterfaceAddressVector& interfaces) override; }; using OsSysCallsSingleton = ThreadSafeSingleton; diff --git a/source/common/api/win32/os_sys_calls_impl.cc b/source/common/api/win32/os_sys_calls_impl.cc index 0ca257243f42e..5d172f88e6c05 100644 --- a/source/common/api/win32/os_sys_calls_impl.cc +++ b/source/common/api/win32/os_sys_calls_impl.cc @@ -409,5 +409,19 @@ SysCallBoolResult OsSysCallsImpl::socketTcpInfo([[maybe_unused]] os_fd_t sockfd, return {false, WSAEOPNOTSUPP}; } +bool OsSysCallsImpl::supportsGetifaddrs() const { + if (alternate_getifaddrs_.has_value()) { + return true; + } + return false; +} + +SysCallIntResult OsSysCallsImpl::getifaddrs([[maybe_unused]] InterfaceAddressVector& interfaces) { + if (alternate_getifaddrs_.has_value()) { + return alternate_getifaddrs_.value()(interfaces); + } + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + } // namespace Api } // namespace Envoy diff --git a/source/common/api/win32/os_sys_calls_impl.h b/source/common/api/win32/os_sys_calls_impl.h index 7b087be9533dc..5268643f47e26 100644 --- a/source/common/api/win32/os_sys_calls_impl.h +++ b/source/common/api/win32/os_sys_calls_impl.h @@ -52,6 +52,8 @@ class OsSysCallsImpl : public OsSysCalls { SysCallSocketResult duplicate(os_fd_t oldfd) override; SysCallSocketResult accept(os_fd_t socket, sockaddr* addr, socklen_t* addrlen) override; SysCallBoolResult socketTcpInfo(os_fd_t sockfd, EnvoyTcpInfo* tcp_info) override; + bool supportsGetifaddrs() const override; + SysCallIntResult getifaddrs(InterfaceAddressVector&) override; }; using OsSysCallsSingleton = ThreadSafeSingleton; diff --git a/source/common/network/BUILD b/source/common/network/BUILD index c6ad2e9651200..1f352e7e59638 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -15,7 +15,6 @@ envoy_cc_library( deps = [ ":socket_interface_lib", "//envoy/network:address_interface", - "//source/common/api:os_sys_calls_lib", "//source/common/common:assert_lib", "//source/common/common:safe_memcpy_lib", "//source/common/common:statusor_lib", diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index 8138b1bcf41a8..80d0aef51bd3b 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -239,36 +239,22 @@ void Utility::throwWithMalformedIp(absl::string_view ip_address) { // need to be updated in the future. Discussion can be found at Github issue #939. Address::InstanceConstSharedPtr Utility::getLocalAddress(const Address::IpVersion version) { Address::InstanceConstSharedPtr ret; -#ifdef SUPPORTS_GETIFADDRS - struct ifaddrs* ifaddr; - struct ifaddrs* ifa; + if (Api::OsSysCallsSingleton::get().supportsGetifaddrs()) { + Api::InterfaceAddressVector interface_addresses{}; - const int rc = getifaddrs(&ifaddr); - RELEASE_ASSERT(!rc, ""); + const Api::SysCallIntResult rc = + Api::OsSysCallsSingleton::get().getifaddrs(interface_addresses); + RELEASE_ASSERT(!rc.return_value_, fmt::format("getiffaddrs error: {}", rc.errno_)); - // man getifaddrs(3) - for (ifa = ifaddr; ifa != nullptr; ifa = ifa->ifa_next) { - if (ifa->ifa_addr == nullptr) { - continue; - } - - if ((ifa->ifa_addr->sa_family == AF_INET && version == Address::IpVersion::v4) || - (ifa->ifa_addr->sa_family == AF_INET6 && version == Address::IpVersion::v6)) { - const struct sockaddr_storage* addr = - reinterpret_cast(ifa->ifa_addr); - ret = Address::addressFromSockAddrOrThrow( - *addr, (version == Address::IpVersion::v4) ? sizeof(sockaddr_in) : sizeof(sockaddr_in6)); - if (!isLoopbackAddress(*ret)) { + // man getifaddrs(3) + for (const auto& interface_address : interface_addresses) { + if (!isLoopbackAddress(*interface_address.interface_addr_)) { + ret = interface_address.interface_addr_; break; } } } - if (ifaddr) { - freeifaddrs(ifaddr); - } -#endif - // If the local address is not found above, then return the loopback address by default. if (ret == nullptr) { if (version == Address::IpVersion::v4) { diff --git a/test/common/api/BUILD b/test/common/api/BUILD new file mode 100644 index 0000000000000..d8d514ba3a41c --- /dev/null +++ b/test/common/api/BUILD @@ -0,0 +1,17 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "os_sys_calls_test", + srcs = ["os_sys_calls_test.cc"], + deps = [ + "//source/common/api:os_sys_calls_lib", + ], +) diff --git a/test/common/api/os_sys_calls_test.cc b/test/common/api/os_sys_calls_test.cc new file mode 100644 index 0000000000000..5748b257ac791 --- /dev/null +++ b/test/common/api/os_sys_calls_test.cc @@ -0,0 +1,34 @@ +#include "source/common/api/os_sys_calls_impl.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +TEST(OsSyscallsTest, SetAlternateGetifaddrs) { + auto& os_syscalls = Api::OsSysCallsSingleton::get(); + const bool pre_alternate_support = os_syscalls.supportsGetifaddrs(); + Api::InterfaceAddressVector interfaces{}; +#if defined(WIN32) || (defined(__ANDROID_API__) && __ANDROID_API__ < 24) + EXPECT_FALSE(pre_alternate_support); + EXPECT_DEATH(os_syscalls.getifaddrs(interfaces), "not implemented"); +#else + EXPECT_TRUE(pre_alternate_support); + const auto pre_alternate_rc = os_syscalls.getifaddrs(interfaces); + EXPECT_EQ(0, pre_alternate_rc.return_value_); + EXPECT_FALSE(interfaces.empty()); +#endif + + os_syscalls.setAlternateGetifaddrs( + [](Api::InterfaceAddressVector& interfaces) -> Api::SysCallIntResult { + interfaces.emplace_back("made_up_if", 0, nullptr); + return {0, 0}; + }); + interfaces.clear(); + + const bool post_alternate_support = os_syscalls.supportsGetifaddrs(); + EXPECT_TRUE(post_alternate_support); + EXPECT_EQ(0, os_syscalls.getifaddrs(interfaces).return_value_); + EXPECT_EQ(1, interfaces.size()); + EXPECT_EQ("made_up_if", interfaces.front().interface_name_); +} +} // namespace Envoy diff --git a/test/common/http/conn_pool_grid_test.cc b/test/common/http/conn_pool_grid_test.cc index 69528416002db..b0d722cac83fd 100644 --- a/test/common/http/conn_pool_grid_test.cc +++ b/test/common/http/conn_pool_grid_test.cc @@ -714,8 +714,24 @@ TEST_F(ConnectivityGridTest, ConnectionCloseDuringCreation) { ASSERT_TRUE(optional_it1.has_value()); EXPECT_EQ("HTTP/3", (**optional_it1)->protocolDescription()); + const bool supports_getifaddrs = Api::OsSysCallsSingleton::get().supportsGetifaddrs(); + Api::InterfaceAddressVector interfaces{}; + if (supports_getifaddrs) { + ASSERT_EQ(0, Api::OsSysCallsSingleton::get().getifaddrs(interfaces).return_value_); + } + Api::MockOsSysCalls os_sys_calls; TestThreadsafeSingletonInjector os_calls(&os_sys_calls); + EXPECT_CALL(os_sys_calls, supportsGetifaddrs()).WillOnce(Return(supports_getifaddrs)); + if (supports_getifaddrs) { + EXPECT_CALL(os_sys_calls, getifaddrs(_)) + .WillOnce( + Invoke([&](Api::InterfaceAddressVector& interface_vector) -> Api::SysCallIntResult { + interface_vector.insert(interface_vector.begin(), interfaces.begin(), + interfaces.end()); + return {0, 0}; + })); + } EXPECT_CALL(os_sys_calls, socket(_, _, _)).WillOnce(Return(Api::SysCallSocketResult{1, 0})); #if defined(__APPLE__) || defined(WIN32) EXPECT_CALL(os_sys_calls, setsocketblocking(1, false)) diff --git a/test/mocks/api/mocks.h b/test/mocks/api/mocks.h index 1df4c36dcabcb..5736dbd28e694 100644 --- a/test/mocks/api/mocks.h +++ b/test/mocks/api/mocks.h @@ -111,6 +111,9 @@ class MockOsSysCalls : public OsSysCallsImpl { MOCK_METHOD(bool, supportsUdpGro, (), (const)); MOCK_METHOD(bool, supportsIpTransparent, (), (const)); MOCK_METHOD(bool, supportsMptcp, (), (const)); + MOCK_METHOD(bool, supportsGetifaddrs, (), (const)); + MOCK_METHOD(void, setAlternateGetifaddrs, (AlternateGetifaddrs alternate_getifaddrs)); + MOCK_METHOD(SysCallIntResult, getifaddrs, (InterfaceAddressVector & interfaces)); // Map from (sockfd,level,optname) to boolean socket option. using SockOptKey = std::tuple; diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 6df79bdbfbf64..80ca75656c1e0 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -11,6 +11,7 @@ ALS AMZ APC API +ARRAYSIZE ARN ASAN ASCII @@ -823,6 +824,7 @@ megamiss mem memcmp memcpy +memset memoize mergeable messagename From d79a3ab49f1aa522d0a465385425e3e00c8db147 Mon Sep 17 00:00:00 2001 From: David Schinazi Date: Fri, 19 Nov 2021 15:09:35 -0800 Subject: [PATCH 07/51] Update QUICHE from 4f552f349 to 81314ecbd (#19066) https://github.com/google/quiche/compare/4f552f349..81314ecbd $ git log 4f552f349..81314ecbd --date=short --no-merges --format="%ad %al %s" 2021-11-17 haoyuewang Internal change 2021-11-16 haoyuewang Deprecate --gfe2_restart_flag_quic_dispatcher_support_multiple_cid_per_connection_v2 2021-11-16 vasilvv Use raw hashes in WebTransportFingerprintProofVerifier. 2021-11-16 wub In TlsServerHandshaker, do not call ProofSourceHandle::SelectCertificate if QUIC connection has disconnected. 2021-11-16 quiche-dev Check for the data_deferred state before writing data for a stream. 2021-11-16 quiche-dev Consolidates references to the third-party nghttp2.h header in a single place. 2021-11-16 quiche-dev Renames Http2ErrorCode::NO_ERROR to Http2ErrorCode::HTTP2_NO_ERROR to avoid conflicts with the builtin Windows macro NO_ERROR. 2021-11-16 dschinazi Deprecate QUIC version T051 2021-11-16 quiche-dev Adds optional RST_STREAM NO_ERROR behavior after sending a fin to an incomplete request as a server. 2021-11-16 wub Deprecate --gfe2_reloadable_flag_quic_tls_restore_connection_context_in_callbacks. 2021-11-15 quiche-dev Let OgHttp2Session avoid visitor callbacks for data on unknown streams. 2021-11-15 quiche-dev Introduce NoOpHeadersHandler in OgHttp2Session. 2021-11-15 quiche-dev Extract NoOpHeadersHandler and HeaderByteListenerInterface into separate header files. 2021-11-15 quiche-dev Add testing to demonstrate nghttp2 and oghttp2 handling of data on a closed stream. 2021-11-15 quiche-dev Consolidates stream close behavior in a smaller number of places. Signed-off-by: David Schinazi --- bazel/external/quiche.BUILD | 23 ++++++ bazel/external/quiche.patch | 99 -------------------------- bazel/repository_locations.bzl | 6 +- source/common/http/http2/codec_impl.cc | 2 +- 4 files changed, 27 insertions(+), 103 deletions(-) diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index a738127058bc9..807089028873c 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -121,6 +121,7 @@ envoy_cc_library( "quiche/http2/adapter/http2_session.h", "quiche/http2/adapter/http2_util.h", "quiche/http2/adapter/http2_visitor_interface.h", + "quiche/http2/adapter/nghttp2.h", "quiche/http2/adapter/nghttp2_adapter.h", "quiche/http2/adapter/nghttp2_callbacks.h", "quiche/http2/adapter/nghttp2_data_provider.h", @@ -146,6 +147,8 @@ envoy_cc_library( ":spdy_core_header_block_lib", ":spdy_core_http2_deframer_lib", ":spdy_core_protocol_lib", + ":spdy_header_byte_listener_interface_lib", + ":spdy_no_op_headers_handler_lib", ], ) @@ -864,6 +867,26 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "spdy_no_op_headers_handler_lib", + hdrs = ["quiche/spdy/core/no_op_headers_handler.h"], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quiche_common_platform", + ], +) + +envoy_cc_library( + name = "spdy_header_byte_listener_interface_lib", + hdrs = ["quiche/spdy/core/header_byte_listener_interface.h"], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quiche_common_platform", + ], +) + envoy_cc_library( name = "spdy_core_alt_svc_wire_format_lib", srcs = ["quiche/spdy/core/spdy_alt_svc_wire_format.cc"], diff --git a/bazel/external/quiche.patch b/bazel/external/quiche.patch index 8d9666a9adeb4..e69de29bb2d1d 100644 --- a/bazel/external/quiche.patch +++ b/bazel/external/quiche.patch @@ -1,99 +0,0 @@ ---- http2/adapter/callback_visitor.h -+++ http2/adapter/callback_visitor.h -@@ -8,6 +8,10 @@ - #include "absl/container/flat_hash_map.h" - #include "http2/adapter/http2_visitor_interface.h" - #include "http2/adapter/nghttp2_util.h" -+ -+// Required to build on Windows. -+typedef ptrdiff_t ssize_t; -+ - #include "third_party/nghttp2/src/lib/includes/nghttp2/nghttp2.h" - #include "common/platform/api/quiche_export.h" - ---- http2/adapter/http2_protocol.h -+++ http2/adapter/http2_protocol.h -@@ -10,6 +10,8 @@ - #include "absl/types/variant.h" - #include "common/platform/api/quiche_export.h" - -+#undef NO_ERROR -+ - namespace http2 { - namespace adapter { - ---- http2/adapter/nghttp2_callbacks.h -+++ http2/adapter/nghttp2_callbacks.h -@@ -5,6 +5,10 @@ - - #include "http2/adapter/http2_protocol.h" - #include "http2/adapter/nghttp2_util.h" -+ -+// Required to build on Windows. -+typedef ptrdiff_t ssize_t; -+ - #include "third_party/nghttp2/src/lib/includes/nghttp2/nghttp2.h" - - namespace http2 { ---- http2/adapter/nghttp2_data_provider.h -+++ http2/adapter/nghttp2_data_provider.h -@@ -5,6 +5,10 @@ - #include - - #include "http2/adapter/data_source.h" -+ -+// Required to build on Windows. -+typedef ptrdiff_t ssize_t; -+ - #include "third_party/nghttp2/src/lib/includes/nghttp2/nghttp2.h" - - namespace http2 { ---- http2/adapter/nghttp2_session.h -+++ http2/adapter/nghttp2_session.h -@@ -5,6 +5,10 @@ - - #include "http2/adapter/http2_session.h" - #include "http2/adapter/nghttp2_util.h" -+ -+// Required to build on Windows. -+typedef ptrdiff_t ssize_t; -+ - #include "third_party/nghttp2/src/lib/includes/nghttp2/nghttp2.h" - #include "common/platform/api/quiche_export.h" - ---- http2/adapter/nghttp2_util.h -+++ http2/adapter/nghttp2_util.h -@@ -10,6 +10,10 @@ - #include "absl/types/span.h" - #include "http2/adapter/data_source.h" - #include "http2/adapter/http2_protocol.h" - #include "http2/adapter/http2_visitor_interface.h" -+ -+// Required to build on Windows. -+typedef ptrdiff_t ssize_t; -+ - #include "third_party/nghttp2/src/lib/includes/nghttp2/nghttp2.h" - #include "spdy/core/spdy_header_block.h" - ---- http2/adapter/http2_util.cc -+++ http2/adapter/http2_util.cc -@@ -1,5 +1,7 @@ - #include "third_party/http2/adapter/http2_util.h" - -+#undef NO_ERROR -+ - namespace http2 { - namespace adapter { - namespace { - ---- http2/adapter/oghttp2_session.cc -+++ http2/adapter/oghttp2_session.cc -@@ -11,6 +11,8 @@ - #include "http2/adapter/oghttp2_util.h" - #include "spdy/core/spdy_protocol.h" - -+#undef NO_ERROR -+ - namespace http2 { - namespace adapter { - diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index b2ed497e3a18a..89485043ef4e4 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -838,12 +838,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "4f552f349b8df000af24bc6cfa0b78fdc2467fef", - sha256 = "355c80803698d2a44363ed304d250059cc5d7712281481803717f8d779229bd8", + version = "81314ecbd8cfd3c8399fcd1c1e7ca557b41d18de", + sha256 = "61e28bf91daba19f65af53b19fbefd5876b3365835e7c0091ccf0c91dba72ba6", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["dataplane_core"], - release_date = "2021-11-15", + release_date = "2021-11-17", cpe = "N/A", ), com_googlesource_googleurl = dict( diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 2193059f02d31..1e19b3c752904 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -902,7 +902,7 @@ int ConnectionImpl::onData(int32_t stream_id, const uint8_t* data, size_t len) { void ConnectionImpl::goAway() { if (use_new_codec_wrapper_) { adapter_->SubmitGoAway(adapter_->GetHighestReceivedStreamId(), - http2::adapter::Http2ErrorCode::NO_ERROR, ""); + http2::adapter::Http2ErrorCode::HTTP2_NO_ERROR, ""); } else { int rc = nghttp2_submit_goaway(session_, NGHTTP2_FLAG_NONE, nghttp2_session_get_last_proc_stream_id(session_), From 91c2e6ba81f6892e4ab132f934d9f6179a50bbc2 Mon Sep 17 00:00:00 2001 From: Arun Olappamanna Vasudevan Date: Mon, 22 Nov 2021 23:49:34 -0800 Subject: [PATCH 08/51] Support for x-forwarded-host header. (#18639) If, for a request, the host/authority header is changed when the request is proxied, set the x-forwarded-host header as: x-forwarded-host = append(x-forwarded-host, host) Risk Level: Low Testing: unit test and manual testing Docs Changes: Added x-forwarded-host header in HTTP header manipulation (configuration/http/http_conn_man/headers). Under the host_rewrite options in route_components.proto (in v3), added that when host is rewritten, the XFH header is appended with the original value of host header if append_x_forwarded_host option is set. Release Notes: router: added support for x-forwarded-host header. Platform Specific Features: N/A Fixes #5940 Signed-off-by: Arun Olappamanna Vasudevan --- .../config/route/v3/route_components.proto | 32 +++++-- .../http/http_conn_man/headers.rst | 19 ++++ docs/root/version_history/current.rst | 1 + envoy/http/header_map.h | 1 + envoy/router/router.h | 5 ++ source/common/http/async_client_impl.h | 1 + source/common/http/utility.cc | 8 ++ source/common/http/utility.h | 8 ++ source/common/router/config_impl.cc | 13 +-- source/common/router/config_impl.h | 3 + source/common/router/delegating_route_impl.cc | 2 + source/common/router/delegating_route_impl.h | 1 + source/common/router/upstream_request.cc | 3 +- test/common/http/utility_test.cc | 86 +++++++++++++++++++ test/common/router/config_impl_test.cc | 10 +++ test/common/router/router_test.cc | 2 + test/mocks/router/mocks.h | 1 + 17 files changed, 185 insertions(+), 11 deletions(-) diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index 90479bac5ee10..0a0a79e119d11 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -617,7 +617,7 @@ message CorsPolicy { core.v3.RuntimeFractionalPercent shadow_enabled = 10; } -// [#next-free-field: 38] +// [#next-free-field: 39] message RouteAction { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RouteAction"; @@ -981,20 +981,29 @@ message RouteAction { oneof host_rewrite_specifier { // Indicates that during forwarding, the host header will be swapped with - // this value. + // this value. Using this option will append the + // :ref:`config_http_conn_man_headers_x-forwarded-host` header if + // :ref:`append_x_forwarded_host ` + // is set. string host_rewrite_literal = 6 [(validate.rules).string = {well_known_regex: HTTP_HEADER_VALUE strict: false}]; // Indicates that during forwarding, the host header will be swapped with // the hostname of the upstream host chosen by the cluster manager. This // option is applicable only when the destination cluster for a route is of - // type *strict_dns* or *logical_dns*. Setting this to true with other cluster - // types has no effect. + // type *strict_dns* or *logical_dns*. Setting this to true with other cluster types + // has no effect. Using this option will append the + // :ref:`config_http_conn_man_headers_x-forwarded-host` header if + // :ref:`append_x_forwarded_host ` + // is set. google.protobuf.BoolValue auto_host_rewrite = 7; // Indicates that during forwarding, the host header will be swapped with the content of given // downstream or :ref:`custom ` header. - // If header value is empty, host header is left intact. + // If header value is empty, host header is left intact. Using this option will append the + // :ref:`config_http_conn_man_headers_x-forwarded-host` header if + // :ref:`append_x_forwarded_host ` + // is set. // // .. attention:: // @@ -1010,6 +1019,10 @@ message RouteAction { // Indicates that during forwarding, the host header will be swapped with // the result of the regex substitution executed on path value with query and fragment removed. // This is useful for transitioning variable content between path segment and subdomain. + // Using this option will append the + // :ref:`config_http_conn_man_headers_x-forwarded-host` header if + // :ref:`append_x_forwarded_host ` + // is set. // // For example with the following config: // @@ -1025,6 +1038,15 @@ message RouteAction { type.matcher.v3.RegexMatchAndSubstitute host_rewrite_path_regex = 35; } + // If set, then a host rewrite action (one of + // :ref:`host_rewrite_literal `, + // :ref:`auto_host_rewrite `, + // :ref:`host_rewrite_header `, or + // :ref:`host_rewrite_path_regex `) + // causes the original value of the host header, if any, to be appended to the + // :ref:`config_http_conn_man_headers_x-forwarded-host` HTTP header. + bool append_x_forwarded_host = 38; + // Specifies the upstream timeout for the route. If not specified, the default is 15s. This // spans between the point at which the entire downstream request (i.e. end-of-stream) has been // processed and when the upstream response has been completely processed. A value of 0 will diff --git a/docs/root/configuration/http/http_conn_man/headers.rst b/docs/root/configuration/http/http_conn_man/headers.rst index 599b0357c73fd..c8b1ab9477156 100644 --- a/docs/root/configuration/http/http_conn_man/headers.rst +++ b/docs/root/configuration/http/http_conn_man/headers.rst @@ -363,6 +363,25 @@ A few very important notes about XFF: XFF is parsed to determine if a request is internal. In this scenario, do not forward XFF and allow Envoy to generate a new one with a single internal origin IP. +.. _config_http_conn_man_headers_x-forwarded-host: + +x-forwarded-host +---------------- + +The *x-forwarded-host* header is a de-facto standard proxy header which indicates the original host +requested by the client in the *:authority* (*host* in HTTP1) header. A compliant proxy *appends* +the original value of the *:authority* header to *x-forwarded-host* only if the *:authority* header +is modified. + +Envoy updates the *:authority* header if a host rewrite option (one of +:ref:`host_rewrite_literal `, +:ref:`auto_host_rewrite `, +:ref:`host_rewrite_header `, or +:ref:`host_rewrite_path_regex `) +is used and appends its original value to *x-forwarded-host* if +:ref:`append_x_forwarded_host ` +is set. + .. _config_http_conn_man_headers_x-forwarded-proto: x-forwarded-proto diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 1ffd6ac11714f..5297fe4e80a2e 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -71,6 +71,7 @@ New Features * listener: added support for :ref:`MPTCP ` (multipath TCP). * oauth filter: added :ref:`cookie_names ` to allow overriding (default) cookie names (``BearerToken``, ``OauthHMAC``, and ``OauthExpires``) set by the filter. * oauth filter: setting IdToken and RefreshToken cookies if they are provided by Identity provider along with AccessToken. +* router: added support for the :ref:`config_http_conn_man_headers_x-forwarded-host` header. * tcp: added a :ref:`FilterState ` :ref:`hash policy `, used by :ref:`TCP proxy ` to allow hashing load balancer algorithms to hash on objects in filter state. * tcp_proxy: added support to populate upstream http connect header values from stream info. * thrift_proxy: add header to metadata filter for turning headers into dynamic metadata. diff --git a/envoy/http/header_map.h b/envoy/http/header_map.h index 3fb5506c84f02..dd004fbd799a7 100644 --- a/envoy/http/header_map.h +++ b/envoy/http/header_map.h @@ -304,6 +304,7 @@ class HeaderEntry { HEADER_FUNC(Expect) \ HEADER_FUNC(ForwardedClientCert) \ HEADER_FUNC(ForwardedFor) \ + HEADER_FUNC(ForwardedHost) \ HEADER_FUNC(ForwardedProto) \ HEADER_FUNC(GrpcTimeout) \ HEADER_FUNC(Host) \ diff --git a/envoy/router/router.h b/envoy/router/router.h index c829262d1403e..31bcd119de35d 100644 --- a/envoy/router/router.h +++ b/envoy/router/router.h @@ -870,6 +870,11 @@ class RouteEntry : public ResponseEntry { */ virtual bool autoHostRewrite() const PURE; + /** + * @return bool true if the x-forwarded-host header should be updated. + */ + virtual bool appendXfh() const PURE; + /** * @return MetadataMatchCriteria* the metadata that a subset load balancer should match when * selecting an upstream host diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index 1e7c7c8ce44d3..3ff696d9f21b0 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -259,6 +259,7 @@ class AsyncStreamImpl : public AsyncClient::Stream, } const Router::VirtualHost& virtualHost() const override { return virtual_host_; } bool autoHostRewrite() const override { return false; } + bool appendXfh() const override { return false; } bool includeVirtualHostRateLimits() const override { return true; } const Router::PathMatchCriterion& pathMatchCriterion() const override { return path_match_criterion_; diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc index 6f2a55da1056a..7d1bc1a8aa312 100644 --- a/source/common/http/utility.cc +++ b/source/common/http/utility.cc @@ -387,6 +387,14 @@ void Utility::appendVia(RequestOrResponseHeaderMap& headers, const std::string& headers.appendVia(via, ", "); } +void Utility::updateAuthority(RequestHeaderMap& headers, absl::string_view hostname, + const bool append_xfh) { + if (append_xfh && !headers.getHostValue().empty()) { + headers.appendForwardedHost(headers.getHostValue(), ","); + } + headers.setHost(hostname); +} + std::string Utility::createSslRedirectPath(const RequestHeaderMap& headers) { ASSERT(headers.Host()); ASSERT(headers.Path()); diff --git a/source/common/http/utility.h b/source/common/http/utility.h index a13922b7aee91..1d4b214da6220 100644 --- a/source/common/http/utility.h +++ b/source/common/http/utility.h @@ -194,6 +194,14 @@ void appendXff(RequestHeaderMap& headers, const Network::Address::Instance& remo */ void appendVia(RequestOrResponseHeaderMap& headers, const std::string& via); +/** + * Update authority with the specified hostname. + * @param headers headers where authority should be updated. + * @param hostname hostname that authority should be updated with. + * @param append_xfh append the original authority to the x-forwarded-host header. + */ +void updateAuthority(RequestHeaderMap& headers, absl::string_view hostname, bool append_xfh); + /** * Creates an SSL (https) redirect path based on the input host and path headers. * @param headers supplies the request headers. diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 705911b97a3ee..e8170860523bf 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -405,7 +405,8 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost, route.route().has_host_rewrite_path_regex() ? route.route().host_rewrite_path_regex().substitution() : ""), - cluster_name_(route.route().cluster()), cluster_header_name_(route.route().cluster_header()), + append_xfh_(route.route().append_x_forwarded_host()), cluster_name_(route.route().cluster()), + cluster_header_name_(route.route().cluster_header()), cluster_not_found_response_code_(ConfigUtility::parseClusterNotFoundResponseCode( route.route().cluster_not_found_response_code())), timeout_(PROTOBUF_GET_MS_OR_DEFAULT(route.route(), timeout, DEFAULT_ROUTE_TIMEOUT_MS)), @@ -670,7 +671,7 @@ void RouteEntryImplBase::finalizeRequestHeaders(Http::RequestHeaderMap& headers, } if (!host_rewrite_.empty()) { - headers.setHost(host_rewrite_); + Http::Utility::updateAuthority(headers, host_rewrite_, append_xfh_); } else if (auto_host_rewrite_header_) { const auto header = headers.get(*auto_host_rewrite_header_); if (!header.empty()) { @@ -678,14 +679,16 @@ void RouteEntryImplBase::finalizeRequestHeaders(Http::RequestHeaderMap& headers, // value is used. const absl::string_view header_value = header[0]->value().getStringView(); if (!header_value.empty()) { - headers.setHost(header_value); + Http::Utility::updateAuthority(headers, header_value, append_xfh_); } } } else if (host_rewrite_path_regex_ != nullptr) { const std::string path(headers.getPathValue()); absl::string_view just_path(Http::PathUtil::removeQueryAndFragment(path)); - headers.setHost( - host_rewrite_path_regex_->replaceAll(just_path, host_rewrite_path_regex_substitution_)); + Http::Utility::updateAuthority( + headers, + host_rewrite_path_regex_->replaceAll(just_path, host_rewrite_path_regex_substitution_), + append_xfh_); } // Handle path rewrite diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 4b6126587d247..6b9aac04a25fe 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -572,6 +572,7 @@ class RouteEntryImplBase : public RouteEntry, } const VirtualHost& virtualHost() const override { return vhost_; } bool autoHostRewrite() const override { return auto_host_rewrite_; } + bool appendXfh() const override { return append_xfh_; } const std::multimap& opaqueConfig() const override { return opaque_config_; } @@ -726,6 +727,7 @@ class RouteEntryImplBase : public RouteEntry, const VirtualHost& virtualHost() const override { return parent_->virtualHost(); } bool autoHostRewrite() const override { return parent_->autoHostRewrite(); } + bool appendXfh() const override { return parent_->appendXfh(); } bool includeVirtualHostRateLimits() const override { return parent_->includeVirtualHostRateLimits(); } @@ -882,6 +884,7 @@ class RouteEntryImplBase : public RouteEntry, const absl::optional auto_host_rewrite_header_; const Regex::CompiledMatcherPtr host_rewrite_path_regex_; const std::string host_rewrite_path_regex_substitution_; + const bool append_xfh_; const std::string cluster_name_; const Http::LowerCaseString cluster_header_name_; const Http::Code cluster_not_found_response_code_; diff --git a/source/common/router/delegating_route_impl.cc b/source/common/router/delegating_route_impl.cc index 422b43248dff6..ddf1899837c2c 100644 --- a/source/common/router/delegating_route_impl.cc +++ b/source/common/router/delegating_route_impl.cc @@ -126,6 +126,8 @@ bool DelegatingRouteEntry::autoHostRewrite() const { return base_route_->routeEntry()->autoHostRewrite(); } +bool DelegatingRouteEntry::appendXfh() const { return base_route_->routeEntry()->appendXfh(); } + const MetadataMatchCriteria* DelegatingRouteEntry::metadataMatchCriteria() const { return base_route_->routeEntry()->metadataMatchCriteria(); } diff --git a/source/common/router/delegating_route_impl.h b/source/common/router/delegating_route_impl.h index f9696e8fc7441..048c21b003185 100644 --- a/source/common/router/delegating_route_impl.h +++ b/source/common/router/delegating_route_impl.h @@ -95,6 +95,7 @@ class DelegatingRouteEntry : public Router::RouteEntry { const VirtualCluster* virtualCluster(const Http::HeaderMap& headers) const override; const VirtualHost& virtualHost() const override; bool autoHostRewrite() const override; + bool appendXfh() const override; const MetadataMatchCriteria* metadataMatchCriteria() const override; const std::multimap& opaqueConfig() const override; bool includeVirtualHostRateLimits() const override; diff --git a/source/common/router/upstream_request.cc b/source/common/router/upstream_request.cc index 213e71cb58a1f..8b9baba46cfde 100644 --- a/source/common/router/upstream_request.cc +++ b/source/common/router/upstream_request.cc @@ -470,7 +470,8 @@ void UpstreamRequest::onPoolReady( calling_encode_headers_ = true; auto* headers = parent_.downstreamHeaders(); if (parent_.routeEntry()->autoHostRewrite() && !host->hostname().empty()) { - parent_.downstreamHeaders()->setHost(host->hostname()); + Http::Utility::updateAuthority(*parent_.downstreamHeaders(), host->hostname(), + parent_.routeEntry()->appendXfh()); } if (span_ != nullptr) { diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index e602f2108df0c..c1263240d26fe 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -364,6 +364,92 @@ TEST(HttpUtility, appendVia) { } } +TEST(HttpUtility, updateAuthority) { + { + TestRequestHeaderMapImpl headers; + Utility::updateAuthority(headers, "dns.name", true); + EXPECT_EQ("dns.name", headers.get_(":authority")); + EXPECT_EQ("", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers; + Utility::updateAuthority(headers, "dns.name", false); + EXPECT_EQ("dns.name", headers.get_(":authority")); + EXPECT_EQ("", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers; + Utility::updateAuthority(headers, "", true); + EXPECT_EQ("", headers.get_(":authority")); + EXPECT_EQ("", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers; + Utility::updateAuthority(headers, "", false); + EXPECT_EQ("", headers.get_(":authority")); + EXPECT_EQ("", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers{{":authority", "host.com"}}; + Utility::updateAuthority(headers, "dns.name", true); + EXPECT_EQ("dns.name", headers.get_(":authority")); + EXPECT_EQ("host.com", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers{{":authority", "host.com"}}; + Utility::updateAuthority(headers, "dns.name", false); + EXPECT_EQ("dns.name", headers.get_(":authority")); + EXPECT_EQ("", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers{{":authority", "host.com"}}; + Utility::updateAuthority(headers, "", true); + EXPECT_EQ("", headers.get_(":authority")); + EXPECT_EQ("host.com", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers{{":authority", "host.com"}}; + Utility::updateAuthority(headers, "", false); + EXPECT_EQ("", headers.get_(":authority")); + EXPECT_EQ("", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers{{":authority", "dns.name"}, {"x-forwarded-host", "host.com"}}; + Utility::updateAuthority(headers, "newhost.com", true); + EXPECT_EQ("newhost.com", headers.get_(":authority")); + EXPECT_EQ("host.com,dns.name", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers{{":authority", "dns.name"}, {"x-forwarded-host", "host.com"}}; + Utility::updateAuthority(headers, "newhost.com", false); + EXPECT_EQ("newhost.com", headers.get_(":authority")); + EXPECT_EQ("host.com", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers{{"x-forwarded-host", "host.com"}}; + Utility::updateAuthority(headers, "dns.name", true); + EXPECT_EQ("dns.name", headers.get_(":authority")); + EXPECT_EQ("host.com", headers.get_("x-forwarded-host")); + } + + { + TestRequestHeaderMapImpl headers{{"x-forwarded-host", "host.com"}}; + Utility::updateAuthority(headers, "dns.name", false); + EXPECT_EQ("dns.name", headers.get_(":authority")); + EXPECT_EQ("host.com", headers.get_("x-forwarded-host")); + } +} + TEST(HttpUtility, createSslRedirectPath) { { TestRequestHeaderMapImpl headers{{":authority", "www.lyft.com"}, {":path", "/hello"}}; diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 8ea8287a1a0d9..3a4ae6e8a19a7 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -649,11 +649,13 @@ TEST_F(RouteMatcherTest, TestRoutes) { route: cluster: ats host_rewrite_header: x-rewrite-host + append_x_forwarded_host: true - match: path: "/do-not-rewrite-host-with-header-value" route: cluster: ats host_rewrite_header: x-rewrite-host + append_x_forwarded_host: true - match: path: "/rewrite-host-with-path-regex/envoyproxy.io" route: @@ -663,6 +665,7 @@ TEST_F(RouteMatcherTest, TestRoutes) { google_re2: {} regex: "^/.+/(.+)$" substitution: \1 + append_x_forwarded_host: true - match: prefix: "/" route: @@ -936,6 +939,9 @@ TEST_F(RouteMatcherTest, TestRoutes) { EXPECT_FALSE(route->currentUrlPathAfterRewrite(headers).has_value()); route->finalizeRequestHeaders(headers, stream_info, true); EXPECT_EQ("new_host", headers.get_(Http::Headers::get().Host)); + // Config setting append_x_forwarded_host is false (by default). Expect empty x-forwarded-host + // header value. + EXPECT_EQ("", headers.get_(Http::Headers::get().ForwardedHost)); } // Rewrites host using supplied header. @@ -946,6 +952,7 @@ TEST_F(RouteMatcherTest, TestRoutes) { EXPECT_FALSE(route->currentUrlPathAfterRewrite(headers).has_value()); route->finalizeRequestHeaders(headers, stream_info, true); EXPECT_EQ("rewrote", headers.get_(Http::Headers::get().Host)); + EXPECT_EQ("api.lyft.com", headers.get_(Http::Headers::get().ForwardedHost)); } // Does not rewrite host because of missing header. @@ -956,6 +963,7 @@ TEST_F(RouteMatcherTest, TestRoutes) { EXPECT_FALSE(route->currentUrlPathAfterRewrite(headers).has_value()); route->finalizeRequestHeaders(headers, stream_info, true); EXPECT_EQ("api.lyft.com", headers.get_(Http::Headers::get().Host)); + EXPECT_EQ("", headers.get_(Http::Headers::get().ForwardedHost)); } // Rewrites host using path. @@ -966,6 +974,7 @@ TEST_F(RouteMatcherTest, TestRoutes) { EXPECT_FALSE(route->currentUrlPathAfterRewrite(headers).has_value()); route->finalizeRequestHeaders(headers, stream_info, true); EXPECT_EQ("envoyproxy.io", headers.get_(Http::Headers::get().Host)); + EXPECT_EQ("api.lyft.com", headers.get_(Http::Headers::get().ForwardedHost)); } // Rewrites host using path, removes query parameters @@ -976,6 +985,7 @@ TEST_F(RouteMatcherTest, TestRoutes) { EXPECT_FALSE(route->currentUrlPathAfterRewrite(headers).has_value()); route->finalizeRequestHeaders(headers, stream_info, true); EXPECT_EQ("envoyproxy.io", headers.get_(Http::Headers::get().Host)); + EXPECT_EQ("api.lyft.com", headers.get_(Http::Headers::get().ForwardedHost)); } // Case sensitive rewrite matching test. diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 105dadbba072b..46506b71174a4 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -5977,6 +5977,7 @@ TEST_F(RouterTest, AutoHostRewriteEnabled) { Http::TestRequestHeaderMapImpl outgoing_headers; HttpTestUtility::addDefaultHeaders(outgoing_headers); outgoing_headers.setHost(cm_.thread_local_cluster_.conn_pool_.host_->hostname_); + outgoing_headers.setForwardedHost(req_host); EXPECT_CALL(callbacks_.route_->route_entry_, timeout()) .WillOnce(Return(std::chrono::milliseconds(0))); @@ -6002,6 +6003,7 @@ TEST_F(RouterTest, AutoHostRewriteEnabled) { EXPECT_EQ(host_address_, host->address()); })); EXPECT_CALL(callbacks_.route_->route_entry_, autoHostRewrite()).WillOnce(Return(true)); + EXPECT_CALL(callbacks_.route_->route_entry_, appendXfh()).WillOnce(Return(true)); router_.decodeHeaders(incoming_headers, true); EXPECT_EQ(1U, callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index e3ea24f69e9d2..e5684f335842c 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -390,6 +390,7 @@ class MockRouteEntry : public RouteEntry { MOCK_METHOD(const VirtualCluster*, virtualCluster, (const Http::HeaderMap& headers), (const)); MOCK_METHOD(const VirtualHost&, virtualHost, (), (const)); MOCK_METHOD(bool, autoHostRewrite, (), (const)); + MOCK_METHOD(bool, appendXfh, (), (const)); MOCK_METHOD((const std::multimap&), opaqueConfig, (), (const)); MOCK_METHOD(bool, includeVirtualHostRateLimits, (), (const)); MOCK_METHOD(const CorsPolicy*, corsPolicy, (), (const)); From 400564b19147d6b12381c4bdca3f8b92ce6078ed Mon Sep 17 00:00:00 2001 From: Ryan Hamilton Date: Tue, 23 Nov 2021 05:16:52 -0800 Subject: [PATCH 09/51] Update QUICHE from 81314ecbd to c2ddf95dc (#19081) * Update QUICHE from 81314ecbd to c2ddf95dc https://github.com/google/quiche/compare/81314ecbd..c2ddf95dc $ git log 81314ecbd..c2ddf95dc --date=short --no-merges --format="%ad %al %s" 2021-11-22 quiche-dev Adds an `emplace()` method to RunOnExit, and deletes the move and copy constructors. 2021-11-22 quiche-dev Makes a safety mechanism related to trailers configurable, and disabled by default. 2021-11-22 haoyuewang Internal change 2021-11-22 haoyuewang Internal change 2021-11-22 fayang Do not reuse tokens received in NEW_TOKEN frames for different connection attempts by: 1) add source address token to QuicClientSessionCache, 2) Clear token after use. 2021-11-19 fayang Add mutable_session_cache() to QuicCryptoClientConfig. 2021-11-19 fayang Move QuicClientSessionCache in chromium to shared code by making following changes: 1) Rename FlushInvalidEntries() and Flush() to RemoveExpiredEntries() and Clear(), respectively. 2) Remove clock_ and SetClockForTesting(), instead, pass in QuicWallTime (use ToUNIXSeconds to get seconds from UNIX epoch because SSL_SESSION_get_time returns seconds from UNIX epoch) to Lookup and RemoveExpiredEntries. 3) Remove memory_pressure_listener_ and OnMemoryPressure(). In chromium, memory_pressure_listener_ and OnMemoryPressure() will be moved to QuicStreamFactory::QuicCryptoClientConfigOwner. 4) Replace base::LRUCache with QuicLRUCache (and add hasher for QuicServerId). 2021-11-19 quiche-dev Move the logic of submitting SETTINGS from OgHttp2Adapter to OgHttp2Session. 2021-11-19 haoyuewang Add QUIC_EXPORT_PRIVATE to RawSha256 method since it is used directly in third_party/quic/quic_transport/web_transport_fingerprint_proof_verifier_test.cc 2021-11-19 fayang Let QuicLRUCache take explicit hasher. Also added various iterators, change Lookup to return iterator and add Erase function. 2021-11-19 quiche-dev Validates that a HEADERS frame with a 100-199 status code does not contain a fin. Signed-off-by: Ryan Hamilton --- bazel/external/quiche.BUILD | 2 + bazel/repository_locations.bzl | 6 +- source/common/quic/BUILD | 12 - .../quic/client_connection_factory_impl.cc | 5 +- .../common/quic/envoy_quic_session_cache.cc | 171 --------- source/common/quic/envoy_quic_session_cache.h | 65 ---- test/common/quic/BUILD | 10 - .../quic/envoy_quic_session_cache_test.cc | 343 ------------------ 8 files changed, 8 insertions(+), 606 deletions(-) delete mode 100644 source/common/quic/envoy_quic_session_cache.cc delete mode 100644 source/common/quic/envoy_quic_session_cache.h delete mode 100644 test/common/quic/envoy_quic_session_cache_test.cc diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 807089028873c..7b7146a0adbcf 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -2003,6 +2003,7 @@ envoy_cc_library( "quiche/quic/core/crypto/curve25519_key_exchange.cc", "quiche/quic/core/crypto/key_exchange.cc", "quiche/quic/core/crypto/p256_key_exchange.cc", + "quiche/quic/core/crypto/quic_client_session_cache.cc", "quiche/quic/core/crypto/quic_compressed_certs_cache.cc", "quiche/quic/core/crypto/quic_crypto_client_config.cc", "quiche/quic/core/crypto/quic_crypto_server_config.cc", @@ -2023,6 +2024,7 @@ envoy_cc_library( "quiche/quic/core/crypto/key_exchange.h", "quiche/quic/core/crypto/p256_key_exchange.h", "quiche/quic/core/crypto/proof_verifier.h", + "quiche/quic/core/crypto/quic_client_session_cache.h", "quiche/quic/core/crypto/quic_compressed_certs_cache.h", "quiche/quic/core/crypto/quic_crypto_client_config.h", "quiche/quic/core/crypto/quic_crypto_server_config.h", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 89485043ef4e4..1279bc5d24e64 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -838,12 +838,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "81314ecbd8cfd3c8399fcd1c1e7ca557b41d18de", - sha256 = "61e28bf91daba19f65af53b19fbefd5876b3365835e7c0091ccf0c91dba72ba6", + version = "c2ddf95dc2380e1d7cd5aa8fe9f61b4e01b23e6b", + sha256 = "435dc3c3858d8328c48fb633bc46431aa5aea2a29e0b5f125f743b0b5b5c1513", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["dataplane_core"], - release_date = "2021-11-17", + release_date = "2021-11-22", cpe = "N/A", ), com_googlesource_googleurl = dict( diff --git a/source/common/quic/BUILD b/source/common/quic/BUILD index a651426c188b2..bab6a0c97cde2 100644 --- a/source/common/quic/BUILD +++ b/source/common/quic/BUILD @@ -124,17 +124,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "envoy_quic_session_cache_lib", - srcs = ["envoy_quic_session_cache.cc"], - hdrs = ["envoy_quic_session_cache.h"], - external_deps = ["quiche_quic_platform"], - tags = ["nofips"], - deps = [ - "@com_github_google_quiche//:quic_core_crypto_crypto_handshake_lib", - ], -) - envoy_cc_library( name = "spdy_server_push_utils_for_envoy_lib", srcs = ["spdy_server_push_utils_for_envoy.cc"], @@ -173,7 +162,6 @@ envoy_cc_library( ":envoy_quic_connection_helper_lib", ":envoy_quic_proof_verifier_lib", ":envoy_quic_server_session_lib", - ":envoy_quic_session_cache_lib", ":envoy_quic_utils_lib", "//envoy/http:codec_interface", "//envoy/registry", diff --git a/source/common/quic/client_connection_factory_impl.cc b/source/common/quic/client_connection_factory_impl.cc index 42a57d4c66f82..c6465aca4453f 100644 --- a/source/common/quic/client_connection_factory_impl.cc +++ b/source/common/quic/client_connection_factory_impl.cc @@ -1,8 +1,9 @@ #include "source/common/quic/client_connection_factory_impl.h" -#include "source/common/quic/envoy_quic_session_cache.h" #include "source/common/quic/quic_transport_socket_factory.h" +#include "quiche/quic/core/crypto/quic_client_session_cache.h" + namespace Envoy { namespace Quic { @@ -34,7 +35,7 @@ std::shared_ptr PersistentQuicInfoImpl::cryptoConf client_context_ = context; client_config_ = std::make_shared( std::make_unique(getContext(transport_socket_factory_)), - std::make_unique((time_source_))); + std::make_unique()); } // Return the latest client config. return client_config_; diff --git a/source/common/quic/envoy_quic_session_cache.cc b/source/common/quic/envoy_quic_session_cache.cc deleted file mode 100644 index b6ce80f7717c6..0000000000000 --- a/source/common/quic/envoy_quic_session_cache.cc +++ /dev/null @@ -1,171 +0,0 @@ -#include "source/common/quic/envoy_quic_session_cache.h" - -namespace Envoy { -namespace Quic { -namespace { - -// This value was chosen arbitrarily. We can make this configurable if needed. -// TODO(14829) ensure this is tested and scaled for upstream. -constexpr size_t MaxSessionCacheEntries = 1024; - -// Returns false if the SSL session doesn't exist or it is expired. -bool isSessionValid(SSL_SESSION* session, SystemTime now) { - if (session == nullptr) { - return false; - } - const time_t now_time_t = std::chrono::system_clock::to_time_t(now); - if (now_time_t < 0) { - return false; - } - const uint64_t now_u64 = static_cast(now_time_t); - const uint64_t session_time = SSL_SESSION_get_time(session); - const uint64_t session_expiration = session_time + SSL_SESSION_get_timeout(session); - // now_u64 may be slightly behind because of differences in how time is calculated at this layer - // versus BoringSSL. Add a second of wiggle room to account for this. - return session_time <= now_u64 + 1 && now_u64 < session_expiration; -} - -bool doApplicationStatesMatch(const quic::ApplicationState* state, - const quic::ApplicationState* other) { - if (state == other) { - return true; - } - if ((state != nullptr && other == nullptr) || (state == nullptr && other != nullptr)) { - return false; - } - return (*state == *other); -} - -} // namespace - -EnvoyQuicSessionCache::EnvoyQuicSessionCache(TimeSource& time_source) : time_source_(time_source) {} - -EnvoyQuicSessionCache::~EnvoyQuicSessionCache() = default; - -void EnvoyQuicSessionCache::Insert(const quic::QuicServerId& server_id, - bssl::UniquePtr session, - const quic::TransportParameters& params, - const quic::ApplicationState* application_state) { - auto it = cache_.find(server_id); - if (it == cache_.end()) { - // New server ID, add a new entry. - createAndInsertEntry(server_id, std::move(session), params, application_state); - return; - } - Entry& entry = it->second; - if (params == *entry.params && - doApplicationStatesMatch(application_state, entry.application_state.get())) { - // The states are both the same, so we only need to insert the session. - entry.pushSession(std::move(session)); - return; - } - // States are different, replace the entry. - cache_.erase(it); - createAndInsertEntry(server_id, std::move(session), params, application_state); -} - -std::unique_ptr -EnvoyQuicSessionCache::Lookup(const quic::QuicServerId& server_id, const SSL_CTX* /*ctx*/) { - auto it = cache_.find(server_id); - if (it == cache_.end()) { - return nullptr; - } - Entry& entry = it->second; - const SystemTime system_time = time_source_.systemTime(); - if (!isSessionValid(entry.peekSession(), system_time)) { - cache_.erase(it); - return nullptr; - } - auto state = std::make_unique(); - state->tls_session = entry.popSession(); - if (entry.params != nullptr) { - state->transport_params = std::make_unique(*entry.params); - } - if (entry.application_state != nullptr) { - state->application_state = std::make_unique(*entry.application_state); - } - return state; -} - -void EnvoyQuicSessionCache::ClearEarlyData(const quic::QuicServerId& server_id) { - auto it = cache_.find(server_id); - if (it == cache_.end()) { - return; - } - for (bssl::UniquePtr& session : it->second.sessions) { - if (session != nullptr) { - session.reset(SSL_SESSION_copy_without_early_data(session.get())); - } - } -} - -void EnvoyQuicSessionCache::prune() { - quic::QuicServerId oldest_id; - uint64_t oldest_expiration = std::numeric_limits::max(); - const SystemTime system_time = time_source_.systemTime(); - auto it = cache_.begin(); - while (it != cache_.end()) { - Entry& entry = it->second; - SSL_SESSION* session = entry.peekSession(); - if (!isSessionValid(session, system_time)) { - it = cache_.erase(it); - } else { - if (cache_.size() >= MaxSessionCacheEntries) { - // Only track the oldest session if we are at the size limit. - const uint64_t session_expiration = - SSL_SESSION_get_time(session) + SSL_SESSION_get_timeout(session); - if (session_expiration < oldest_expiration) { - oldest_expiration = session_expiration; - oldest_id = it->first; - } - } - ++it; - } - } - if (cache_.size() >= MaxSessionCacheEntries) { - cache_.erase(oldest_id); - } -} - -void EnvoyQuicSessionCache::createAndInsertEntry(const quic::QuicServerId& server_id, - bssl::UniquePtr session, - const quic::TransportParameters& params, - const quic::ApplicationState* application_state) { - prune(); - ASSERT(cache_.size() < MaxSessionCacheEntries); - Entry entry; - entry.pushSession(std::move(session)); - entry.params = std::make_unique(params); - if (application_state != nullptr) { - entry.application_state = std::make_unique(*application_state); - } - cache_.insert(std::make_pair(server_id, std::move(entry))); -} - -size_t EnvoyQuicSessionCache::size() const { return cache_.size(); } - -EnvoyQuicSessionCache::Entry::Entry() = default; -EnvoyQuicSessionCache::Entry::Entry(Entry&&) noexcept = default; -EnvoyQuicSessionCache::Entry::~Entry() = default; - -void EnvoyQuicSessionCache::Entry::pushSession(bssl::UniquePtr session) { - if (sessions[0] != nullptr) { - sessions[1] = std::move(sessions[0]); - } - sessions[0] = std::move(session); -} - -bssl::UniquePtr EnvoyQuicSessionCache::Entry::popSession() { - if (sessions[0] == nullptr) { - return nullptr; - } - bssl::UniquePtr session = std::move(sessions[0]); - sessions[0] = std::move(sessions[1]); - sessions[1] = nullptr; - return session; -} - -SSL_SESSION* EnvoyQuicSessionCache::Entry::peekSession() { return sessions[0].get(); } - -} // namespace Quic -} // namespace Envoy diff --git a/source/common/quic/envoy_quic_session_cache.h b/source/common/quic/envoy_quic_session_cache.h deleted file mode 100644 index c15fe0d0e658e..0000000000000 --- a/source/common/quic/envoy_quic_session_cache.h +++ /dev/null @@ -1,65 +0,0 @@ -#pragma once - -#include "envoy/common/time.h" - -#include "quiche/quic/core/crypto/quic_crypto_client_config.h" - -namespace Envoy { -namespace Quic { - -// Implementation of quic::SessionCache using an Envoy time source. -class EnvoyQuicSessionCache : public quic::SessionCache { -public: - explicit EnvoyQuicSessionCache(TimeSource& time_source); - ~EnvoyQuicSessionCache() override; - - // From quic::SessionCache. - void Insert(const quic::QuicServerId& server_id, bssl::UniquePtr session, - const quic::TransportParameters& params, - const quic::ApplicationState* application_state) override; - std::unique_ptr Lookup(const quic::QuicServerId& server_id, - const SSL_CTX* ctx) override; - void ClearEarlyData(const quic::QuicServerId& server_id) override; - - // Returns number of entries in the cache. - size_t size() const; - -private: - struct Entry { - Entry(); - Entry(Entry&&) noexcept; - ~Entry(); - - // Adds a new session onto sessions, dropping the oldest one if two are - // already stored. - void pushSession(bssl::UniquePtr session); - - // Retrieves and removes the latest session from the entry. - bssl::UniquePtr popSession(); - - SSL_SESSION* peekSession(); - - // We only save the last two sessions per server as that is sufficient in practice. This is - // because we only need one to create a new connection, and that new connection should send - // us another ticket. We keep two instead of one in case that connection attempt fails. - bssl::UniquePtr sessions[2]; - std::unique_ptr params; - std::unique_ptr application_state; - }; - - // Remove all entries that are no longer valid. If all entries were valid but the cache is at its - // size limit, instead remove the oldest entry. This walks the entire list of entries. - void prune(); - - // Creates a new entry and insert into cache_. This walks the entire list of entries. - void createAndInsertEntry(const quic::QuicServerId& server_id, - bssl::UniquePtr session, - const quic::TransportParameters& params, - const quic::ApplicationState* application_state); - - std::map cache_; - TimeSource& time_source_; -}; - -} // namespace Quic -} // namespace Envoy diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD index 604eaf1cd8f24..f549b51982aec 100644 --- a/test/common/quic/BUILD +++ b/test/common/quic/BUILD @@ -80,16 +80,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "envoy_quic_session_cache_test", - srcs = ["envoy_quic_session_cache_test.cc"], - external_deps = ["quiche_quic_platform"], - tags = ["nofips"], - deps = [ - "//source/common/quic:envoy_quic_session_cache_lib", - ], -) - envoy_cc_test( name = "envoy_quic_proof_verifier_test", srcs = ["envoy_quic_proof_verifier_test.cc"], diff --git a/test/common/quic/envoy_quic_session_cache_test.cc b/test/common/quic/envoy_quic_session_cache_test.cc deleted file mode 100644 index c7601331597fb..0000000000000 --- a/test/common/quic/envoy_quic_session_cache_test.cc +++ /dev/null @@ -1,343 +0,0 @@ -#include "source/common/quic/envoy_quic_session_cache.h" - -#include "absl/strings/escaping.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -namespace Envoy { -namespace Quic { -namespace { - -constexpr uint32_t Timeout = 1000; -const quic::QuicVersionLabel FakeVersionLabel = 0x01234567; -const quic::QuicVersionLabel FakeVersionLabel2 = 0x89ABCDEF; -const uint64_t FakeIdleTimeoutMilliseconds = 12012; -const uint8_t FakeStatelessResetTokenData[16] = {0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F}; -const uint64_t FakeMaxPacketSize = 9001; -const uint64_t FakeInitialMaxData = 101; -const bool FakeDisableMigration = true; -const auto CustomParameter1 = static_cast(0xffcd); -const char* CustomParameter1Value = "foo"; -const auto CustomParameter2 = static_cast(0xff34); -const char* CustomParameter2Value = "bar"; - -std::vector createFakeStatelessResetToken() { - return std::vector(FakeStatelessResetTokenData, - FakeStatelessResetTokenData + sizeof(FakeStatelessResetTokenData)); -} - -// Make a TransportParameters that has a few fields set to help test comparison. -std::unique_ptr makeFakeTransportParams() { - auto params = std::make_unique(); - params->perspective = quic::Perspective::IS_CLIENT; - params->legacy_version_information = quic::TransportParameters::LegacyVersionInformation(); - params->legacy_version_information->version = FakeVersionLabel; - params->legacy_version_information->supported_versions.push_back(FakeVersionLabel); - params->legacy_version_information->supported_versions.push_back(FakeVersionLabel2); - params->max_idle_timeout_ms.set_value(FakeIdleTimeoutMilliseconds); - params->stateless_reset_token = createFakeStatelessResetToken(); - params->max_udp_payload_size.set_value(FakeMaxPacketSize); - params->initial_max_data.set_value(FakeInitialMaxData); - params->disable_active_migration = FakeDisableMigration; - params->custom_parameters[CustomParameter1] = CustomParameter1Value; - params->custom_parameters[CustomParameter2] = CustomParameter2Value; - return params; -} - -// Generated with SSL_SESSION_to_bytes. -static constexpr char CachedSession[] = - "3082068702010102020304040213010420b9c2a657e565db0babd09e192a9fc4d768fbd706" - "9f03f9278a4a0be62392e55b0420d87ed2ab8cafc986fd2e288bd2d654cd57c3a2bed1d532" - "20726e55fed39d021ea10602045ed16771a205020302a300a382025f3082025b30820143a0" - "03020102020104300d06092a864886f70d01010b0500302c3110300e060355040a13074163" - "6d6520436f311830160603550403130f496e7465726d656469617465204341301e170d3133" - "303130313130303030305a170d3233313233313130303030305a302d3110300e060355040a" - "130741636d6520436f3119301706035504031310746573742e6578616d706c652e636f6d30" - "59301306072a8648ce3d020106082a8648ce3d030107034200040526220e77278300d06bc0" - "86aff4f999a828a2ed5cc75adc2972794befe885aa3a9b843de321b36b0a795289cebff1a5" - "428bad5e34665ce5e36daad08fb3ffd8a3523050300e0603551d0f0101ff04040302078030" - "130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301b06" - "03551d11041430128210746573742e6578616d706c652e636f6d300d06092a864886f70d01" - "010b050003820101008c1f1e380831b6437a8b9284d28d4ead38d9503a9fc936db89048aa2" - "edd6ec2fb830d962ef7a4f384e679504f4d5520f3272e0b9e702b110aff31711578fa5aeb1" - "11e9d184c994b0f97e7b17d1995f3f477f25bc1258398ec0ec729caed55d594a009f48093a" - "17f33a7f3bb6e420cc3499838398a421d93c7132efa8bee5ed2645cbc55179c400da006feb" - "761badd356cac3bd7a0e6b22a511106a355ec62a4c0ac2541d2996adb4a918c866d10c3e31" - "62039a91d4ce600b276740d833380b37f66866d261bf6efa8855e7ae6c7d12a8a864cd9a1f" - "4663e07714b0204e51bbc189a2d04c2a5043202379ff1c8cbf30cbb44fde4ee9a1c0c976dc" - "4943df2c132ca4020400aa7f047d494e534543555245003072020101020203040402130104" - "000420d87ed2ab8cafc986fd2e288bd2d654cd57c3a2bed1d53220726e55fed39d021ea106" - "02045ed16771a205020302a300a4020400b20302011db5060404bd909308b807020500ffff" - "ffffb9050203093a80ba07040568332d3238bb030101ffbc03040100b20302011db3820307" - "30820303308201eba003020102020102300d06092a864886f70d01010b050030243110300e" - "060355040a130741636d6520436f3110300e06035504031307526f6f74204341301e170d31" - "33303130313130303030305a170d3233313233313130303030305a302c3110300e06035504" - "0a130741636d6520436f311830160603550403130f496e7465726d65646961746520434130" - "820122300d06092a864886f70d01010105000382010f003082010a0282010100cd3550e70a" - "6880e52bf0012b93110c50f723e1d8d2ed489aea3b649f82fae4ad2396a8a19b31d1d64ab2" - "79f1c18003184154a5303a82bd57109cfd5d34fd19d3211bcb06e76640e1278998822dd72e" - "0d5c059a740d45de325e784e81b4c86097f08b2a8ce057f6b9db5a53641d27e09347d993ee" - "acf67be7d297b1a6853775ffaaf78fae924e300b5654fd32f99d3cd82e95f56417ff26d265" - "e2b1786c835d67a4d8ae896b6eb34b35a5b1033c209779ed0bf8de25a13a507040ae9e0475" - "a26a2f15845b08c3e0554e47dbbc7925b02e580dbcaaa6f2eecde6b8028c5b00b33d44d0a6" - "bfb3e72e9d4670de45d1bd79bdc0f2470b71286091c29873152db4b1f30203010001a33830" - "36300e0603551d0f0101ff04040302020430130603551d25040c300a06082b060105050703" - "01300f0603551d130101ff040530030101ff300d06092a864886f70d01010b050003820101" - "00bc4f8234860558dd404a626403819bfc759029d625a002143e75ebdb2898d1befdd326c3" - "4b14dc3507d732bb29af7e6af31552db53052a2be0d950efee5e0f699304231611ed8bf73a" - "6f216a904c6c2f1a2186d1ed08a8005a7914394d71e7d4b643c808f86365c5fecad8b52934" - "2d3b3f03447126d278d75b1dab3ed53f23e36e9b3d695f28727916e5ee56ce22d387c81f05" - "919b2a37bd4981eb67d9f57b7072285dbbb61f48b6b14768c069a092aad5a094cf295dafd2" - "3ca008f89a5f5ab37a56e5f68df45091c7cb85574677127087a2887ba3baa6d4fc436c6e40" - "40885e81621d38974f0c7f0d792418c5adebb10e92a165f8d79b169617ff575c0d4a85b506" - "0404bd909308b603010100b70402020403b807020500ffffffffb9050203093a80ba070405" - "68332d3238bb030101ff"; - -class FakeTimeSource : public TimeSource { -public: - // From TimeSource. - SystemTime systemTime() override { return fake_time_; } - MonotonicTime monotonicTime() override { - // EnvoyQuicSessionCache does not use monotonic time, return empty value. - ADD_FAILURE() << "Unexpected call to monotonicTime"; - return MonotonicTime(); - } - - void advance(int seconds) { fake_time_ += std::chrono::seconds(seconds); } - -private: - SystemTime fake_time_{}; -}; - -} // namespace - -class EnvoyQuicSessionCacheTest : public ::testing::Test { -public: - EnvoyQuicSessionCacheTest() - : ssl_ctx_(SSL_CTX_new(TLS_method())), cache_(time_source_), - params_(makeFakeTransportParams()) {} - -protected: - bssl::UniquePtr makeSession(uint32_t timeout = Timeout) { - std::string cached_session = - absl::HexStringToBytes(absl::string_view(CachedSession, sizeof(CachedSession))); - SSL_SESSION* session = - SSL_SESSION_from_bytes(reinterpret_cast(cached_session.data()), - cached_session.size(), ssl_ctx_.get()); - SSL_SESSION_set_time(session, std::chrono::system_clock::to_time_t(time_source_.systemTime())); - SSL_SESSION_set_timeout(session, timeout); - return bssl::UniquePtr(session); - } - - bssl::UniquePtr ssl_ctx_; - FakeTimeSource time_source_; - EnvoyQuicSessionCache cache_; - std::unique_ptr params_; -}; - -// Tests that simple insertion and lookup work correctly. -TEST_F(EnvoyQuicSessionCacheTest, SingleSession) { - bssl::UniquePtr session = makeSession(); - quic::QuicServerId id1("a.com", 443); - - std::unique_ptr params2 = makeFakeTransportParams(); - bssl::UniquePtr session2 = makeSession(); - SSL_SESSION* unowned2 = session2.get(); - quic::QuicServerId id2("b.com", 443); - - EXPECT_EQ(nullptr, cache_.Lookup(id1, ssl_ctx_.get())); - EXPECT_EQ(nullptr, cache_.Lookup(id2, ssl_ctx_.get())); - EXPECT_EQ(0u, cache_.size()); - - cache_.Insert(id1, std::move(session), *params_, nullptr); - EXPECT_EQ(1u, cache_.size()); - std::unique_ptr resumption_state = cache_.Lookup(id1, ssl_ctx_.get()); - ASSERT_NE(resumption_state, nullptr); - ASSERT_NE(resumption_state->transport_params, nullptr); - EXPECT_EQ(*params_, *(resumption_state->transport_params)); - EXPECT_EQ(nullptr, cache_.Lookup(id2, ssl_ctx_.get())); - // No session is available for id1, even though the entry exists. - EXPECT_EQ(1u, cache_.size()); - EXPECT_EQ(nullptr, cache_.Lookup(id1, ssl_ctx_.get())); - // Lookup() will trigger a deletion of invalid entry. - EXPECT_EQ(0u, cache_.size()); - - bssl::UniquePtr session3 = makeSession(); - SSL_SESSION* unowned3 = session3.get(); - quic::QuicServerId id3("c.com", 443); - cache_.Insert(id3, std::move(session3), *params_, nullptr); - cache_.Insert(id2, std::move(session2), *params2, nullptr); - EXPECT_EQ(2u, cache_.size()); - std::unique_ptr resumption_state2 = cache_.Lookup(id2, ssl_ctx_.get()); - ASSERT_NE(resumption_state2, nullptr); - EXPECT_EQ(unowned2, resumption_state2->tls_session.get()); - std::unique_ptr resumption_state3 = cache_.Lookup(id3, ssl_ctx_.get()); - ASSERT_NE(resumption_state3, nullptr); - EXPECT_EQ(unowned3, resumption_state3->tls_session.get()); - - // Verify that the cache is cleared after Lookups. - EXPECT_EQ(nullptr, cache_.Lookup(id1, ssl_ctx_.get())); - EXPECT_EQ(nullptr, cache_.Lookup(id2, ssl_ctx_.get())); - EXPECT_EQ(nullptr, cache_.Lookup(id3, ssl_ctx_.get())); - EXPECT_EQ(0u, cache_.size()); -} - -TEST_F(EnvoyQuicSessionCacheTest, MultipleSessions) { - bssl::UniquePtr session = makeSession(); - quic::QuicServerId id1("a.com", 443); - bssl::UniquePtr session2 = makeSession(); - SSL_SESSION* unowned2 = session2.get(); - bssl::UniquePtr session3 = makeSession(); - SSL_SESSION* unowned3 = session3.get(); - - cache_.Insert(id1, std::move(session), *params_, nullptr); - cache_.Insert(id1, std::move(session2), *params_, nullptr); - cache_.Insert(id1, std::move(session3), *params_, nullptr); - // The latest session is popped first. - std::unique_ptr resumption_state1 = cache_.Lookup(id1, ssl_ctx_.get()); - ASSERT_NE(resumption_state1, nullptr); - EXPECT_EQ(unowned3, resumption_state1->tls_session.get()); - std::unique_ptr resumption_state2 = cache_.Lookup(id1, ssl_ctx_.get()); - ASSERT_NE(resumption_state2, nullptr); - EXPECT_EQ(unowned2, resumption_state2->tls_session.get()); - // Only two sessions are cache. - EXPECT_EQ(nullptr, cache_.Lookup(id1, ssl_ctx_.get())); -} - -// Test that when a different TransportParameter is inserted for -// the same server id, the existing entry is removed. -TEST_F(EnvoyQuicSessionCacheTest, DifferentTransportParams) { - bssl::UniquePtr session = makeSession(); - quic::QuicServerId id1("a.com", 443); - bssl::UniquePtr session2 = makeSession(); - bssl::UniquePtr session3 = makeSession(); - SSL_SESSION* unowned3 = session3.get(); - - cache_.Insert(id1, std::move(session), *params_, nullptr); - cache_.Insert(id1, std::move(session2), *params_, nullptr); - // tweak the transport parameters a little bit. - params_->perspective = quic::Perspective::IS_SERVER; - cache_.Insert(id1, std::move(session3), *params_, nullptr); - std::unique_ptr resumption_state = cache_.Lookup(id1, ssl_ctx_.get()); - ASSERT_NE(resumption_state, nullptr); - EXPECT_EQ(unowned3, resumption_state->tls_session.get()); - EXPECT_EQ(*params_.get(), *resumption_state->transport_params); - EXPECT_EQ(nullptr, cache_.Lookup(id1, ssl_ctx_.get())); -} - -TEST_F(EnvoyQuicSessionCacheTest, DifferentApplicationState) { - bssl::UniquePtr session = makeSession(); - quic::QuicServerId id1("a.com", 443); - bssl::UniquePtr session2 = makeSession(); - bssl::UniquePtr session3 = makeSession(); - SSL_SESSION* unowned3 = session3.get(); - quic::ApplicationState state; - state.push_back('a'); - - cache_.Insert(id1, std::move(session), *params_, &state); - cache_.Insert(id1, std::move(session2), *params_, &state); - cache_.Insert(id1, std::move(session3), *params_, nullptr); - std::unique_ptr resumption_state = cache_.Lookup(id1, ssl_ctx_.get()); - ASSERT_NE(resumption_state, nullptr); - EXPECT_EQ(unowned3, resumption_state->tls_session.get()); - EXPECT_EQ(nullptr, resumption_state->application_state); - EXPECT_EQ(nullptr, cache_.Lookup(id1, ssl_ctx_.get())); -} - -TEST_F(EnvoyQuicSessionCacheTest, BothStatesDifferent) { - bssl::UniquePtr session = makeSession(); - quic::QuicServerId id1("a.com", 443); - bssl::UniquePtr session2 = makeSession(); - bssl::UniquePtr session3 = makeSession(); - SSL_SESSION* unowned3 = session3.get(); - quic::ApplicationState state; - state.push_back('a'); - - cache_.Insert(id1, std::move(session), *params_, &state); - cache_.Insert(id1, std::move(session2), *params_, &state); - params_->perspective = quic::Perspective::IS_SERVER; - cache_.Insert(id1, std::move(session3), *params_, nullptr); - std::unique_ptr resumption_state = cache_.Lookup(id1, ssl_ctx_.get()); - ASSERT_NE(resumption_state, nullptr); - EXPECT_EQ(unowned3, resumption_state->tls_session.get()); - EXPECT_EQ(*params_, *resumption_state->transport_params); - EXPECT_EQ(nullptr, resumption_state->application_state); - EXPECT_EQ(nullptr, cache_.Lookup(id1, ssl_ctx_.get())); -} - -// When the size limit is exceeded, the oldest entry should be erased. -TEST_F(EnvoyQuicSessionCacheTest, SizeLimit) { - constexpr size_t size_limit = 1024; - std::array unowned_sessions; - for (size_t i = 0; i <= size_limit; i++) { - time_source_.advance(1); - bssl::UniquePtr session = makeSession(/*timeout=*/10000); - unowned_sessions[i] = session.get(); - quic::QuicServerId id(absl::StrCat("domain", i, ".example.com"), 443); - cache_.Insert(id, std::move(session), *params_, nullptr); - } - EXPECT_EQ(cache_.size(), size_limit); - // First entry has been removed. - quic::QuicServerId id0("domain0.example.com", 443); - EXPECT_EQ(nullptr, cache_.Lookup(id0, ssl_ctx_.get())); - // All other entries all present. - for (size_t i = 1; i <= size_limit; i++) { - quic::QuicServerId id(absl::StrCat("domain", i, ".example.com"), 443); - std::unique_ptr resumption_state = cache_.Lookup(id, ssl_ctx_.get()); - ASSERT_NE(resumption_state, nullptr) << i; - EXPECT_EQ(resumption_state->tls_session.get(), unowned_sessions[i]); - } -} - -TEST_F(EnvoyQuicSessionCacheTest, ClearEarlyData) { - SSL_CTX_set_early_data_enabled(ssl_ctx_.get(), 1); - bssl::UniquePtr session = makeSession(); - quic::QuicServerId id1("a.com", 443); - bssl::UniquePtr session2 = makeSession(); - - EXPECT_TRUE(SSL_SESSION_early_data_capable(session.get())); - EXPECT_TRUE(SSL_SESSION_early_data_capable(session2.get())); - - cache_.Insert(id1, std::move(session), *params_, nullptr); - cache_.Insert(id1, std::move(session2), *params_, nullptr); - - cache_.ClearEarlyData(id1); - - std::unique_ptr resumption_state = cache_.Lookup(id1, ssl_ctx_.get()); - ASSERT_NE(resumption_state, nullptr); - EXPECT_FALSE(SSL_SESSION_early_data_capable(resumption_state->tls_session.get())); - resumption_state = cache_.Lookup(id1, ssl_ctx_.get()); - EXPECT_FALSE(SSL_SESSION_early_data_capable(resumption_state->tls_session.get())); - EXPECT_EQ(nullptr, cache_.Lookup(id1, ssl_ctx_.get())); -} - -// Expired session isn't considered valid and nullptr will be returned upon -// Lookup. -TEST_F(EnvoyQuicSessionCacheTest, Expiration) { - bssl::UniquePtr session = makeSession(); - quic::QuicServerId id1("a.com", 443); - - bssl::UniquePtr session2 = makeSession(3 * Timeout); - SSL_SESSION* unowned2 = session2.get(); - quic::QuicServerId id2("b.com", 443); - - cache_.Insert(id1, std::move(session), *params_, nullptr); - cache_.Insert(id2, std::move(session2), *params_, nullptr); - - EXPECT_EQ(2u, cache_.size()); - // Expire the session. - time_source_.advance(Timeout * 2); - // The entry has not been removed yet. - EXPECT_EQ(2u, cache_.size()); - - EXPECT_EQ(nullptr, cache_.Lookup(id1, ssl_ctx_.get())); - EXPECT_EQ(1u, cache_.size()); - std::unique_ptr resumption_state = cache_.Lookup(id2, ssl_ctx_.get()); - ASSERT_NE(resumption_state, nullptr); - EXPECT_EQ(unowned2, resumption_state->tls_session.get()); - EXPECT_EQ(1u, cache_.size()); -} - -} // namespace Quic -} // namespace Envoy From 5c83001814cd9f57f2179f53940a806b82225e33 Mon Sep 17 00:00:00 2001 From: Hu Shuai Date: Wed, 24 Nov 2021 02:36:40 +0800 Subject: [PATCH 10/51] Fix a small typo (#19058) Signed-off-by: Hu Shuai --- source/common/upstream/load_balancer_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index 5cd361b1da589..f9aad90169781 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -959,7 +959,7 @@ double EdfLoadBalancerBase::applySlowStartFactor(double host_weight, const Host& aggression_ = aggression_runtime_ != absl::nullopt ? aggression_runtime_.value().value() : 1.0; if (aggression_ < 0.0) { ENVOY_LOG_EVERY_POW_2(error, "Invalid runtime value provided for aggression parameter, " - "agression cannot be less than 0.0"); + "aggression cannot be less than 0.0"); } aggression_ = std::max(0.0, aggression_); From 25bf30156731777bf60e3f801773f08b38cf02b2 Mon Sep 17 00:00:00 2001 From: Peter Jausovec Date: Wed, 24 Nov 2021 00:07:48 -0800 Subject: [PATCH 11/51] Fix a broken example in Lua filter docs (#19086) Signed-off-by: Peter Jausovec --- docs/root/configuration/http/http_filters/lua_filter.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/root/configuration/http/http_filters/lua_filter.rst b/docs/root/configuration/http/http_filters/lua_filter.rst index 78777d4e9c7b6..4e55541f9a05a 100644 --- a/docs/root/configuration/http/http_filters/lua_filter.rst +++ b/docs/root/configuration/http/http_filters/lua_filter.rst @@ -769,8 +769,8 @@ its keys can only be *string* or *numeric*. function envoy_on_request(request_handle) local headers = request_handle:headers() request_handle:streamInfo():dynamicMetadata():set("envoy.filters.http.lua", "request.info", { - auth: headers:get("authorization"), - token: headers:get("x-request-token"), + auth = headers:get("authorization"), + token = headers:get("x-request-token"), }) end From bb95af848c939cfe5b5ee33c5b1770558077e64e Mon Sep 17 00:00:00 2001 From: pradeepcrao <84025829+pradeepcrao@users.noreply.github.com> Date: Wed, 24 Nov 2021 14:28:02 +0000 Subject: [PATCH 12/51] Specify type for matching Subject Alternative Name. (#18628) Signed-off-by: Pradeep Rao --- .../transport_sockets/tls/v3/common.proto | 41 ++++++++-- .../tls/v3/tls_spiffe_validator_config.proto | 2 +- configs/envoy_double_proxy.template.yaml | 12 ++- .../envoy_service_to_service.template.yaml | 12 ++- .../arch_overview/security/_include/ssl.yaml | 6 +- .../root/intro/arch_overview/security/ssl.rst | 2 +- .../_include/envoy-demo-tls-client-auth.yaml | 6 +- .../_include/envoy-demo-tls-validation.yaml | 6 +- docs/root/start/quick-start/securing.rst | 12 +-- docs/root/version_history/current.rst | 2 + .../certificate_validation_context_config.h | 3 +- examples/double-proxy/envoy-backend.yaml | 6 +- examples/double-proxy/envoy-frontend.yaml | 6 +- source/common/ssl/BUILD | 1 + ...tificate_validation_context_config_impl.cc | 35 ++++++++- ...rtificate_validation_context_config_impl.h | 9 ++- .../tls/cert_validator/BUILD | 6 ++ .../tls/cert_validator/default_validator.cc | 28 +++---- .../tls/cert_validator/default_validator.h | 18 ++--- .../tls/cert_validator/san_matcher.cc | 53 +++++++++++++ .../tls/cert_validator/san_matcher.h | 51 ++++++++++++ .../cert_validator/spiffe/spiffe_validator.cc | 21 ++--- .../cert_validator/spiffe/spiffe_validator.h | 4 +- .../quic/envoy_quic_proof_source_test.cc | 3 +- .../quic/envoy_quic_proof_verifier_test.cc | 3 +- test/common/secret/sds_api_test.cc | 18 ++++- .../common/secret/secret_manager_impl_test.cc | 37 +++++++++ test/common/upstream/upstream_impl_test.cc | 40 +++++++--- test/config/utility.cc | 4 +- test/config/utility.h | 7 +- .../tls/cert_validator/BUILD | 13 ++++ .../cert_validator/default_validator_test.cc | 77 +++++++++++++------ .../tls/cert_validator/san_matcher_test.cc | 57 ++++++++++++++ .../spiffe_validator_integration_test.cc | 28 ++++++- .../spiffe_validator_integration_test.h | 3 +- .../spiffe/spiffe_validator_test.cc | 28 ++++++- .../tls/cert_validator/test_common.h | 8 +- .../tls/context_impl_test.cc | 35 ++++++++- .../transport_sockets/tls/ssl_socket_test.cc | 30 +++++--- test/integration/ads_integration.cc | 5 +- test/integration/ssl_utility.cc | 19 ++++- test/integration/xfcc_integration_test.cc | 34 +++++--- test/mocks/ssl/mocks.h | 5 +- test/per_file_coverage.sh | 2 +- .../listener_manager_impl_quic_only_test.cc | 40 +++++++--- 45 files changed, 666 insertions(+), 172 deletions(-) create mode 100644 source/extensions/transport_sockets/tls/cert_validator/san_matcher.cc create mode 100644 source/extensions/transport_sockets/tls/cert_validator/san_matcher.h create mode 100644 test/extensions/transport_sockets/tls/cert_validator/san_matcher_test.cc diff --git a/api/envoy/extensions/transport_sockets/tls/v3/common.proto b/api/envoy/extensions/transport_sockets/tls/v3/common.proto index 369ee4d657eb3..c638578392a9b 100644 --- a/api/envoy/extensions/transport_sockets/tls/v3/common.proto +++ b/api/envoy/extensions/transport_sockets/tls/v3/common.proto @@ -9,6 +9,7 @@ import "envoy/type/matcher/v3/string.proto"; import "google/protobuf/any.proto"; import "google/protobuf/wrappers.proto"; +import "envoy/annotations/deprecation.proto"; import "udpa/annotations/migrate.proto"; import "udpa/annotations/sensitive.proto"; import "udpa/annotations/status.proto"; @@ -268,7 +269,26 @@ message CertificateProviderPluginInstance { string certificate_name = 2; } -// [#next-free-field: 15] +// Matcher for subject alternative names, to match both type and value of the SAN. +message SubjectAltNameMatcher { + // Indicates the choice of GeneralName as defined in section 4.2.1.5 of RFC 5280 to match + // against. + enum SanType { + SAN_TYPE_UNSPECIFIED = 0; + EMAIL = 1; + DNS = 2; + URI = 3; + IP_ADDRESS = 4; + } + + // Specification of type of SAN. Note that the default enum value is an invalid choice. + SanType san_type = 1 [(validate.rules).enum = {defined_only: true not_in: 0}]; + + // Matcher for SAN value. + type.matcher.v3.StringMatcher matcher = 2 [(validate.rules).message = {required: true}]; +} + +// [#next-free-field: 16] message CertificateValidationContext { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.auth.CertificateValidationContext"; @@ -298,8 +318,8 @@ message CertificateValidationContext { // `, // :ref:`verify_certificate_hash // `, or - // :ref:`match_subject_alt_names - // `) is also + // :ref:`match_typed_subject_alt_names + // `) is also // specified. // // It can optionally contain certificate revocation lists, in which case Envoy will verify @@ -406,6 +426,8 @@ message CertificateValidationContext { // An optional list of Subject Alternative name matchers. If specified, Envoy will verify that the // Subject Alternative Name of the presented certificate matches one of the specified matchers. + // The matching uses "any" semantics, that is to say, the SAN is verified if at least one matcher is + // matched. // // When a certificate has wildcard DNS SAN entries, to match a specific client, it should be // configured with exact match type in the :ref:`string matcher `. @@ -414,15 +436,22 @@ message CertificateValidationContext { // // .. code-block:: yaml // - // match_subject_alt_names: - // exact: "api.example.com" + // match_typed_subject_alt_names: + // - san_type: DNS + // matcher: + // exact: "api.example.com" // // .. attention:: // // Subject Alternative Names are easily spoofable and verifying only them is insecure, // therefore this option must be used together with :ref:`trusted_ca // `. - repeated type.matcher.v3.StringMatcher match_subject_alt_names = 9; + repeated SubjectAltNameMatcher match_typed_subject_alt_names = 15; + + // This field is deprecated in favor of ref:`match_typed_subject_alt_names + // ` + repeated type.matcher.v3.StringMatcher match_subject_alt_names = 9 + [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; // [#not-implemented-hide:] Must present signed certificate time-stamp. google.protobuf.BoolValue require_signed_certificate_timestamp = 6; diff --git a/api/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.proto b/api/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.proto index cfb5e5c07e90c..382fe985daffd 100644 --- a/api/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.proto +++ b/api/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.proto @@ -42,7 +42,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Note that SPIFFE validator inherits and uses the following options from :ref:`CertificateValidationContext `. // // - :ref:`allow_expired_certificate ` to allow expired certificates. -// - :ref:`match_subject_alt_names ` to match **URI** SAN of certificates. Unlike the default validator, SPIFFE validator only matches **URI** SAN (which equals to SVID in SPIFFE terminology) and ignore other SAN types. +// - :ref:`match_typed_subject_alt_names ` to match **URI** SAN of certificates. Unlike the default validator, SPIFFE validator only matches **URI** SAN (which equals to SVID in SPIFFE terminology) and ignore other SAN types. // message SPIFFECertValidatorConfig { message TrustDomain { diff --git a/configs/envoy_double_proxy.template.yaml b/configs/envoy_double_proxy.template.yaml index b574d6a518c5c..56f4d17ad201a 100644 --- a/configs/envoy_double_proxy.template.yaml +++ b/configs/envoy_double_proxy.template.yaml @@ -149,8 +149,10 @@ static_resources: validation_context: trusted_ca: filename: certs/cacert.pem - match_subject_alt_names: - exact: "front-proxy.yourcompany.net" + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: "front-proxy.yourcompany.net" typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions @@ -188,8 +190,10 @@ static_resources: validation_context: trusted_ca: filename: certs/cacert.pem - match_subject_alt_names: - exact: "collector-grpc.lightstep.com" + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: "collector-grpc.lightstep.com" flags_path: "/etc/envoy/flags" stats_sinks: - name: envoy.stat_sinks.statsd diff --git a/configs/envoy_service_to_service.template.yaml b/configs/envoy_service_to_service.template.yaml index 6ec2f0bde9057..7bdd6a4d0fadc 100644 --- a/configs/envoy_service_to_service.template.yaml +++ b/configs/envoy_service_to_service.template.yaml @@ -350,8 +350,10 @@ static_resources: trusted_ca: filename: certs/cacert.pem {% if host.get('verify_subject_alt_name', False) %} - match_subject_alt_names: - exact: "{{host['verify_subject_alt_name'] }}" + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: "{{host['verify_subject_alt_name'] }}" {% endif %} {% if host.get('sni', False) %} sni: "{{ host['sni'] }}" @@ -520,8 +522,10 @@ static_resources: validation_context: trusted_ca: filename: certs/cacert.pem - match_subject_alt_names: - exact: "collector-grpc.lightstep.com" + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: "collector-grpc.lightstep.com" - name: cds_cluster connect_timeout: 0.25s type: STRICT_DNS diff --git a/docs/root/intro/arch_overview/security/_include/ssl.yaml b/docs/root/intro/arch_overview/security/_include/ssl.yaml index 6f666e4d92a55..00bd6083239bd 100644 --- a/docs/root/intro/arch_overview/security/_include/ssl.yaml +++ b/docs/root/intro/arch_overview/security/_include/ssl.yaml @@ -50,7 +50,9 @@ static_resources: private_key: {"filename": "certs/serverkey.pem"} ocsp_staple: {"filename": "certs/server_ocsp_resp.der"} validation_context: - match_subject_alt_names: - - exact: "foo" + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: "foo" trusted_ca: filename: /etc/ssl/certs/ca-certificates.crt diff --git a/docs/root/intro/arch_overview/security/ssl.rst b/docs/root/intro/arch_overview/security/ssl.rst index e1192833232c9..8878c1faf4190 100644 --- a/docs/root/intro/arch_overview/security/ssl.rst +++ b/docs/root/intro/arch_overview/security/ssl.rst @@ -76,7 +76,7 @@ Example configuration */etc/ssl/certs/ca-certificates.crt* is the default path for the system CA bundle on Debian systems. :ref:`trusted_ca ` along with -:ref:`match_subject_alt_names ` +:ref:`match_typed_subject_alt_names ` makes Envoy verify the server identity of *127.0.0.1:1234* as "foo" in the same way as e.g. cURL does on standard Debian installations. Common paths for system CA bundles on Linux and BSD are: diff --git a/docs/root/start/quick-start/_include/envoy-demo-tls-client-auth.yaml b/docs/root/start/quick-start/_include/envoy-demo-tls-client-auth.yaml index 84367c637f686..dd48870fbda62 100644 --- a/docs/root/start/quick-start/_include/envoy-demo-tls-client-auth.yaml +++ b/docs/root/start/quick-start/_include/envoy-demo-tls-client-auth.yaml @@ -34,8 +34,10 @@ static_resources: validation_context: trusted_ca: filename: certs/cacert.pem - match_subject_alt_names: - - exact: proxy-postgres-frontend.example.com + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: proxy-postgres-frontend.example.com tls_certificates: - certificate_chain: filename: certs/servercert.pem diff --git a/docs/root/start/quick-start/_include/envoy-demo-tls-validation.yaml b/docs/root/start/quick-start/_include/envoy-demo-tls-validation.yaml index b9ad6cc0635e9..054f79e55f366 100644 --- a/docs/root/start/quick-start/_include/envoy-demo-tls-validation.yaml +++ b/docs/root/start/quick-start/_include/envoy-demo-tls-validation.yaml @@ -48,5 +48,7 @@ static_resources: validation_context: trusted_ca: filename: certs/cacert.pem - match_subject_alt_names: - - exact: proxy-postgres-backend.example.com + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: proxy-postgres-backend.example.com diff --git a/docs/root/start/quick-start/securing.rst b/docs/root/start/quick-start/securing.rst index ccfd6bd0ce060..35ff830450a60 100644 --- a/docs/root/start/quick-start/securing.rst +++ b/docs/root/start/quick-start/securing.rst @@ -100,7 +100,7 @@ certificate is valid for. .. note:: If the "Subject Alternative Names" for a certificate are for a wildcard domain, eg ``*.example.com``, - this is what you should use when matching with ``match_subject_alt_names``. + this is what you should use when matching with ``match_typed_subject_alt_names``. .. note:: @@ -122,20 +122,20 @@ and specify a mutually trusted certificate authority: :language: yaml :linenos: :lineno-start: 27 - :lines: 27-39 + :lines: 27-41 :emphasize-lines: 6, 8-10 :caption: :download:`envoy-demo-tls-client-auth.yaml <_include/envoy-demo-tls-client-auth.yaml>` You can further restrict the authentication of connecting clients by specifying the allowed "Subject Alternative Names" in -:ref:`match_subject_alt_names `, +:ref:`match_typed_subject_alt_names `, similar to validating upstream certificates :ref:`described above `. .. literalinclude:: _include/envoy-demo-tls-client-auth.yaml :language: yaml :linenos: :lineno-start: 27 - :lines: 27-39 + :lines: 27-41 :emphasize-lines: 7, 11-12 :caption: :download:`envoy-demo-tls-client-auth.yaml <_include/envoy-demo-tls-client-auth.yaml>` @@ -154,8 +154,8 @@ When connecting to an upstream with client certificates you can set them as foll .. literalinclude:: _include/envoy-demo-tls-client-auth.yaml :language: yaml :linenos: - :lineno-start: 44 - :lines: 44-68 + :lineno-start: 46 + :lines: 46-70 :emphasize-lines: 20-25 :caption: :download:`envoy-demo-tls-client-auth.yaml <_include/envoy-demo-tls-client-auth.yaml>` diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 5297fe4e80a2e..1937c5298b410 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -80,6 +80,7 @@ New Features * thrift_proxy: add host level success/error metrics where success is a reply of type success and error is any other response to a call. * thrift_proxy: support header flags. * thrift_proxy: support subset lb when using request or route metadata. +* tls: added support for :ref:`match_typed_subject_alt_names ` for subject alternative names to enforce specifying the subject alternative name type for the matcher to prevent matching against an unintended type in the certificate. * tls: added support for only verifying the leaf CRL in the certificate chain with :ref:`only_verify_leaf_cert_crl `. * tls: support loading certificate chain and private key via :ref:`pkcs12 `. * tls_inspector filter: added :ref:`enable_ja3_fingerprinting ` to create JA3 fingerprint hash from Client Hello message. @@ -95,4 +96,5 @@ Deprecated * bootstrap: :ref:`dns_resolution_config ` is deprecated in favor of :ref:`typed_dns_resolver_config `. * cluster: :ref:`dns_resolution_config ` is deprecated in favor of :ref:`typed_dns_resolver_config `. * dns_cache: :ref:`dns_resolution_config ` is deprecated in favor of :ref:`typed_dns_resolver_config `. +* tls: :ref:`match_subject_alt_names ` has been deprecated in favor of the :ref:`match_typed_subject_alt_names `. * dns_filter: :ref:`dns_resolution_config ` is deprecated in favor of :ref:`typed_dns_resolver_config `. diff --git a/envoy/ssl/certificate_validation_context_config.h b/envoy/ssl/certificate_validation_context_config.h index d8a1bf6b0af20..d331c45ce254b 100644 --- a/envoy/ssl/certificate_validation_context_config.h +++ b/envoy/ssl/certificate_validation_context_config.h @@ -7,6 +7,7 @@ #include "envoy/api/api.h" #include "envoy/common/pure.h" #include "envoy/extensions/transport_sockets/tls/v3/cert.pb.h" +#include "envoy/extensions/transport_sockets/tls/v3/common.pb.h" #include "envoy/type/matcher/v3/string.pb.h" #include "absl/types/optional.h" @@ -43,7 +44,7 @@ class CertificateValidationContextConfig { /** * @return The subject alt name matchers to be verified, if enabled. */ - virtual const std::vector& + virtual const std::vector& subjectAltNameMatchers() const PURE; /** diff --git a/examples/double-proxy/envoy-backend.yaml b/examples/double-proxy/envoy-backend.yaml index 07cc1a7905f1d..0636354c3771a 100644 --- a/examples/double-proxy/envoy-backend.yaml +++ b/examples/double-proxy/envoy-backend.yaml @@ -26,8 +26,10 @@ static_resources: private_key: filename: certs/serverkey.pem validation_context: - match_subject_alt_names: - - exact: proxy-postgres-frontend.example.com + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: proxy-postgres-frontend.example.com trusted_ca: filename: certs/cacert.pem diff --git a/examples/double-proxy/envoy-frontend.yaml b/examples/double-proxy/envoy-frontend.yaml index 37acbf334124e..21fa643e62ede 100644 --- a/examples/double-proxy/envoy-frontend.yaml +++ b/examples/double-proxy/envoy-frontend.yaml @@ -36,7 +36,9 @@ static_resources: private_key: filename: certs/clientkey.pem validation_context: - match_subject_alt_names: - - exact: proxy-postgres-backend.example.com + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: proxy-postgres-backend.example.com trusted_ca: filename: certs/cacert.pem diff --git a/source/common/ssl/BUILD b/source/common/ssl/BUILD index 714c426b930f3..4526fe428f73f 100644 --- a/source/common/ssl/BUILD +++ b/source/common/ssl/BUILD @@ -32,6 +32,7 @@ envoy_cc_library( "//envoy/ssl:certificate_validation_context_config_interface", "//source/common/common:empty_string", "//source/common/config:datasource_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", ], diff --git a/source/common/ssl/certificate_validation_context_config_impl.cc b/source/common/ssl/certificate_validation_context_config_impl.cc index 564325c3fb662..376f2b65a085e 100644 --- a/source/common/ssl/certificate_validation_context_config_impl.cc +++ b/source/common/ssl/certificate_validation_context_config_impl.cc @@ -1,10 +1,13 @@ #include "source/common/ssl/certificate_validation_context_config_impl.h" #include "envoy/common/exception.h" +#include "envoy/config/core/v3/extension.pb.h" #include "envoy/extensions/transport_sockets/tls/v3/cert.pb.h" +#include "envoy/extensions/transport_sockets/tls/v3/common.pb.h" #include "source/common/common/empty_string.h" #include "source/common/common/fmt.h" +#include "source/common/common/logger.h" #include "source/common/config/datasource.h" namespace Envoy { @@ -22,8 +25,7 @@ CertificateValidationContextConfigImpl::CertificateValidationContextConfigImpl( certificate_revocation_list_path_( Config::DataSource::getPath(config.crl()) .value_or(certificate_revocation_list_.empty() ? EMPTY_STRING : INLINE_STRING)), - subject_alt_name_matchers_(config.match_subject_alt_names().begin(), - config.match_subject_alt_names().end()), + subject_alt_name_matchers_(getSubjectAltNameMatchers(config)), verify_certificate_hash_list_(config.verify_certificate_hash().begin(), config.verify_certificate_hash().end()), verify_certificate_spki_list_(config.verify_certificate_spki().begin(), @@ -51,5 +53,34 @@ CertificateValidationContextConfigImpl::CertificateValidationContextConfigImpl( } } +std::vector +CertificateValidationContextConfigImpl::getSubjectAltNameMatchers( + const envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext& config) { + if (!config.match_typed_subject_alt_names().empty() && + !config.match_subject_alt_names().empty()) { + throw EnvoyException("SAN-based verification using both match_typed_subject_alt_names and " + "the deprecated match_subject_alt_names is not allowed"); + } + std::vector + subject_alt_name_matchers(config.match_typed_subject_alt_names().begin(), + config.match_typed_subject_alt_names().end()); + // Handle deprecated string type san matchers without san type specified, by + // creating a matcher for each supported type. + for (const envoy::type::matcher::v3::StringMatcher& matcher : config.match_subject_alt_names()) { + static constexpr std::array< + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::SanType, 4> + san_types{envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS, + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::URI, + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::EMAIL, + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::IP_ADDRESS}; + for (const auto san_type : san_types) { + subject_alt_name_matchers.emplace_back(); + subject_alt_name_matchers.back().set_san_type(san_type); + *subject_alt_name_matchers.back().mutable_matcher() = matcher; + } + } + return subject_alt_name_matchers; +} + } // namespace Ssl } // namespace Envoy diff --git a/source/common/ssl/certificate_validation_context_config_impl.h b/source/common/ssl/certificate_validation_context_config_impl.h index 038abe08b1e96..6e00605ff5cd4 100644 --- a/source/common/ssl/certificate_validation_context_config_impl.h +++ b/source/common/ssl/certificate_validation_context_config_impl.h @@ -4,6 +4,7 @@ #include "envoy/api/api.h" #include "envoy/extensions/transport_sockets/tls/v3/cert.pb.h" +#include "envoy/extensions/transport_sockets/tls/v3/common.pb.h" #include "envoy/ssl/certificate_validation_context_config.h" #include "envoy/type/matcher/v3/string.pb.h" @@ -24,7 +25,7 @@ class CertificateValidationContextConfigImpl : public CertificateValidationConte const std::string& certificateRevocationListPath() const final { return certificate_revocation_list_path_; } - const std::vector& + const std::vector& subjectAltNameMatchers() const override { return subject_alt_name_matchers_; } @@ -51,11 +52,15 @@ class CertificateValidationContextConfigImpl : public CertificateValidationConte bool onlyVerifyLeafCertificateCrl() const override { return only_verify_leaf_cert_crl_; } private: + static std::vector + getSubjectAltNameMatchers( + const envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext& config); const std::string ca_cert_; const std::string ca_cert_path_; const std::string certificate_revocation_list_; const std::string certificate_revocation_list_path_; - const std::vector subject_alt_name_matchers_; + const std::vector + subject_alt_name_matchers_; const std::vector verify_certificate_hash_list_; const std::vector verify_certificate_spki_list_; const bool allow_expired_certificate_; diff --git a/source/extensions/transport_sockets/tls/cert_validator/BUILD b/source/extensions/transport_sockets/tls/cert_validator/BUILD index ce92df41e80c3..93f745f8faa38 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/BUILD +++ b/source/extensions/transport_sockets/tls/cert_validator/BUILD @@ -13,12 +13,14 @@ envoy_cc_library( srcs = [ "default_validator.cc", "factory.cc", + "san_matcher.cc", "utility.cc", ], hdrs = [ "cert_validator.h", "default_validator.h", "factory.h", + "san_matcher.h", "utility.h", ], external_deps = [ @@ -35,9 +37,13 @@ envoy_cc_library( "//source/common/common:hex_lib", "//source/common/common:minimal_logger_lib", "//source/common/common:utility_lib", + "//source/common/config:utility_lib", "//source/common/stats:symbol_table_lib", "//source/common/stats:utility_lib", "//source/extensions/transport_sockets/tls:stats_lib", "//source/extensions/transport_sockets/tls:utility_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", + "@envoy_api//envoy/type/matcher/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc b/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc index e638b98b42841..93bbe8b17a041 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc +++ b/source/extensions/transport_sockets/tls/cert_validator/default_validator.cc @@ -18,6 +18,7 @@ #include "source/common/common/hex.h" #include "source/common/common/matchers.h" #include "source/common/common/utility.h" +#include "source/common/config/utility.h" #include "source/common/network/address_impl.h" #include "source/common/protobuf/utility.h" #include "source/common/runtime/runtime_features.h" @@ -147,9 +148,9 @@ int DefaultCertValidator::initializeSslContexts(std::vector contexts, const Envoy::Ssl::CertificateValidationContextConfig* cert_validation_config = config_; if (cert_validation_config != nullptr) { if (!cert_validation_config->subjectAltNameMatchers().empty()) { - for (const envoy::type::matcher::v3::StringMatcher& matcher : + for (const envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher& matcher : cert_validation_config->subjectAltNameMatchers()) { - subject_alt_name_matchers_.push_back(Matchers::StringMatcherImpl(matcher)); + subject_alt_name_matchers_.emplace_back(createStringSanMatcher(matcher)); } verify_mode = verify_mode_validation_context; } @@ -221,8 +222,8 @@ int DefaultCertValidator::doVerifyCertChain( // If `trusted_ca` exists, it is already verified in the code above. Thus, we just need to make // sure the verification for other validation context configurations doesn't fail (i.e. either - // `NotValidated` or `Validated`). If `trusted_ca` doesn't exist, we will need to make sure other - // configurations are verified and the verification succeed. + // `NotValidated` or `Validated`). If `trusted_ca` doesn't exist, we will need to make sure + // other configurations are verified and the verification succeed. int validation_status = verify_trusted_ca_ ? validated != Envoy::Ssl::ClientValidationStatus::Failed : validated == Envoy::Ssl::ClientValidationStatus::Validated; @@ -232,8 +233,7 @@ int DefaultCertValidator::doVerifyCertChain( Envoy::Ssl::ClientValidationStatus DefaultCertValidator::verifyCertificate( X509* cert, const std::vector& verify_san_list, - const std::vector>& - subject_alt_name_matchers) { + const std::vector& subject_alt_name_matchers) { Envoy::Ssl::ClientValidationStatus validated = Envoy::Ssl::ClientValidationStatus::NotValidated; if (!verify_san_list.empty()) { @@ -291,23 +291,15 @@ bool DefaultCertValidator::verifySubjectAltName(X509* cert, } bool DefaultCertValidator::matchSubjectAltName( - X509* cert, - const std::vector>& - subject_alt_name_matchers) { + X509* cert, const std::vector& subject_alt_name_matchers) { bssl::UniquePtr san_names( static_cast(X509_get_ext_d2i(cert, NID_subject_alt_name, nullptr, nullptr))); if (san_names == nullptr) { return false; } - for (const GENERAL_NAME* general_name : san_names.get()) { - const std::string san = Utility::generalNameAsString(general_name); - for (auto& config_san_matcher : subject_alt_name_matchers) { - // For DNS SAN, if the StringMatcher type is exact, we have to follow DNS matching semantics. - if (general_name->type == GEN_DNS && - config_san_matcher.matcher().match_pattern_case() == - envoy::type::matcher::v3::StringMatcher::MatchPatternCase::kExact - ? Utility::dnsNameMatch(config_san_matcher.matcher().exact(), absl::string_view(san)) - : config_san_matcher.match(san)) { + for (const auto& config_san_matcher : subject_alt_name_matchers) { + for (const GENERAL_NAME* general_name : san_names.get()) { + if (config_san_matcher->match(general_name)) { return true; } } diff --git a/source/extensions/transport_sockets/tls/cert_validator/default_validator.h b/source/extensions/transport_sockets/tls/cert_validator/default_validator.h index 0cfe2766d1370..64f52bc38728f 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/default_validator.h +++ b/source/extensions/transport_sockets/tls/cert_validator/default_validator.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "envoy/common/pure.h" @@ -18,6 +19,7 @@ #include "source/common/common/matchers.h" #include "source/common/stats/symbol_table_impl.h" #include "source/extensions/transport_sockets/tls/cert_validator/cert_validator.h" +#include "source/extensions/transport_sockets/tls/cert_validator/san_matcher.h" #include "source/extensions/transport_sockets/tls/stats.h" #include "absl/synchronization/mutex.h" @@ -53,10 +55,9 @@ class DefaultCertValidator : public CertValidator, Logger::Loggable& verify_san_list, - const std::vector>& - subject_alt_name_matchers); + Envoy::Ssl::ClientValidationStatus + verifyCertificate(X509* cert, const std::vector& verify_san_list, + const std::vector& subject_alt_name_matchers); /** * Verifies certificate hash for pinning. The hash is a hex-encoded SHA-256 of the DER-encoded @@ -94,10 +95,8 @@ class DefaultCertValidator : public CertValidator, Logger::Loggable>& - subject_alt_name_matchers); + static bool matchSubjectAltName(X509* cert, + const std::vector& subject_alt_name_matchers); private: const Envoy::Ssl::CertificateValidationContextConfig* config_; @@ -107,8 +106,7 @@ class DefaultCertValidator : public CertValidator, Logger::Loggable ca_cert_; std::string ca_file_path_; - std::vector> - subject_alt_name_matchers_; + std::vector subject_alt_name_matchers_; std::vector> verify_certificate_hash_list_; std::vector> verify_certificate_spki_list_; bool verify_trusted_ca_{false}; diff --git a/source/extensions/transport_sockets/tls/cert_validator/san_matcher.cc b/source/extensions/transport_sockets/tls/cert_validator/san_matcher.cc new file mode 100644 index 0000000000000..4be1c64ec096b --- /dev/null +++ b/source/extensions/transport_sockets/tls/cert_validator/san_matcher.cc @@ -0,0 +1,53 @@ +#include "source/extensions/transport_sockets/tls/cert_validator/san_matcher.h" + +#include + +#include "envoy/config/core/v3/extension.pb.h" +#include "envoy/extensions/transport_sockets/tls/v3/common.pb.h" +#include "envoy/registry/registry.h" +#include "envoy/ssl/certificate_validation_context_config.h" + +#include "source/extensions/transport_sockets/tls/utility.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { + +bool StringSanMatcher::match(const GENERAL_NAME* general_name) const { + if (general_name->type != general_name_type_) { + return false; + } + // For DNS SAN, if the StringMatcher type is exact, we have to follow DNS matching semantics. + const std::string san = Utility::generalNameAsString(general_name); + return general_name->type == GEN_DNS && + matcher_.matcher().match_pattern_case() == + envoy::type::matcher::v3::StringMatcher::MatchPatternCase::kExact + ? Utility::dnsNameMatch(matcher_.matcher().exact(), absl::string_view(san)) + : matcher_.match(san); +} + +SanMatcherPtr createStringSanMatcher( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher const& matcher) { + // Verify that a new san type has not been added. + static_assert(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::SanType_MAX == + 4); + + switch (matcher.san_type()) { + case envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS: + return SanMatcherPtr{std::make_unique(GEN_DNS, matcher.matcher())}; + case envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::EMAIL: + return SanMatcherPtr{std::make_unique(GEN_EMAIL, matcher.matcher())}; + case envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::URI: + return SanMatcherPtr{std::make_unique(GEN_URI, matcher.matcher())}; + case envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::IP_ADDRESS: + return SanMatcherPtr{std::make_unique(GEN_IPADD, matcher.matcher())}; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/tls/cert_validator/san_matcher.h b/source/extensions/transport_sockets/tls/cert_validator/san_matcher.h new file mode 100644 index 0000000000000..5c2141f4b5707 --- /dev/null +++ b/source/extensions/transport_sockets/tls/cert_validator/san_matcher.h @@ -0,0 +1,51 @@ +#pragma once + +#include + +#include "envoy/config/core/v3/extension.pb.h" +#include "envoy/extensions/transport_sockets/tls/v3/common.pb.h" +#include "envoy/ssl/certificate_validation_context_config.h" +#include "envoy/type/matcher/v3/string.pb.h" + +#include "source/common/common/hash.h" +#include "source/common/common/matchers.h" +#include "source/common/protobuf/protobuf.h" +#include "source/extensions/transport_sockets/tls/utility.h" + +#include "openssl/x509v3.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { + +/** Interface to verify if there is a match in a list of subject alternative + * names. + */ +class SanMatcher { +public: + virtual bool match(GENERAL_NAME const*) const PURE; + virtual ~SanMatcher() = default; +}; + +using SanMatcherPtr = std::unique_ptr; + +class StringSanMatcher : public SanMatcher { +public: + bool match(const GENERAL_NAME* general_name) const override; + ~StringSanMatcher() override = default; + StringSanMatcher(int general_name_type, envoy::type::matcher::v3::StringMatcher matcher) + : general_name_type_(general_name_type), matcher_(matcher) {} + +private: + const int general_name_type_; + const Matchers::StringMatcherImpl matcher_; +}; + +SanMatcherPtr createStringSanMatcher( + const envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher& matcher); + +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.cc b/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.cc index 63df8297c76dc..c3b8d338fcae3 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.cc +++ b/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.cc @@ -1,5 +1,6 @@ #include "source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.h" +#include "envoy/extensions/transport_sockets/tls/v3/common.pb.h" #include "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.pb.h" #include "envoy/network/transport_socket.h" #include "envoy/registry/registry.h" @@ -37,7 +38,14 @@ SPIFFEValidator::SPIFFEValidator(const Envoy::Ssl::CertificateValidationContextC if (!config->subjectAltNameMatchers().empty()) { for (const auto& matcher : config->subjectAltNameMatchers()) { - subject_alt_name_matchers_.push_back(Matchers::StringMatcherImpl(matcher)); + if (matcher.san_type() == + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::URI) { + // Only match against URI SAN since SPIFFE specification does not restrict values in other + // SAN types. See the discussion: https://github.com/envoyproxy/envoy/issues/15392 + // TODO(pradeepcrao): Throw an exception when a non-URI matcher is encountered after the + // deprecated field match_subject_alt_names is removed + subject_alt_name_matchers_.emplace_back(createStringSanMatcher(matcher)); + } } } @@ -224,15 +232,10 @@ bool SPIFFEValidator::matchSubjectAltName(X509& leaf_cert) { ASSERT(san_names != nullptr, "san_names should have at least one name after SPIFFE cert validation"); - // Only match against URI SAN since SPIFFE specification does not restrict values in other SAN - // types. See the discussion: https://github.com/envoyproxy/envoy/issues/15392 for (const GENERAL_NAME* general_name : san_names.get()) { - if (general_name->type == GEN_URI) { - const std::string san = Utility::generalNameAsString(general_name); - for (const auto& config_san_matcher : subject_alt_name_matchers_) { - if (config_san_matcher.match(san)) { - return true; - } + for (const auto& config_san_matcher : subject_alt_name_matchers_) { + if (config_san_matcher->match(general_name)) { + return true; } } } diff --git a/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.h b/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.h index b4cc068e908e5..669d77aec7d05 100644 --- a/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.h +++ b/source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.h @@ -17,6 +17,7 @@ #include "source/common/common/matchers.h" #include "source/common/stats/symbol_table_impl.h" #include "source/extensions/transport_sockets/tls/cert_validator/cert_validator.h" +#include "source/extensions/transport_sockets/tls/cert_validator/san_matcher.h" #include "source/extensions/transport_sockets/tls/stats.h" #include "openssl/ssl.h" @@ -67,8 +68,7 @@ class SPIFFEValidator : public CertValidator { bool allow_expired_certificate_{false}; std::vector> ca_certs_; std::string ca_file_name_; - std::vector> - subject_alt_name_matchers_{}; + std::vector subject_alt_name_matchers_{}; absl::flat_hash_map trust_bundle_stores_; SslStats& stats_; diff --git a/test/common/quic/envoy_quic_proof_source_test.cc b/test/common/quic/envoy_quic_proof_source_test.cc index b481768eb91fa..3c49ad23f35e8 100644 --- a/test/common/quic/envoy_quic_proof_source_test.cc +++ b/test/common/quic/envoy_quic_proof_source_test.cc @@ -58,7 +58,8 @@ class SignatureVerifier { ON_CALL(cert_validation_ctx_config_, certificateRevocationListPath()) .WillByDefault(ReturnRef(path_string)); const std::vector empty_string_list; - const std::vector san_matchers; + const std::vector + san_matchers; ON_CALL(cert_validation_ctx_config_, subjectAltNameMatchers()) .WillByDefault(ReturnRef(san_matchers)); ON_CALL(cert_validation_ctx_config_, verifyCertificateHashList()) diff --git a/test/common/quic/envoy_quic_proof_verifier_test.cc b/test/common/quic/envoy_quic_proof_verifier_test.cc index dea9daa038f5c..06b3405b7d2a9 100644 --- a/test/common/quic/envoy_quic_proof_verifier_test.cc +++ b/test/common/quic/envoy_quic_proof_verifier_test.cc @@ -75,7 +75,8 @@ class EnvoyQuicProofVerifierTest : public testing::Test { const std::string path_string_{"some_path"}; const std::string alpn_{"h2,http/1.1"}; const std::string sig_algs_{"rsa_pss_rsae_sha256"}; - const std::vector san_matchers_; + const std::vector + san_matchers_; const std::string empty_string_; const std::vector empty_string_list_; const std::string cert_chain_{quic::test::kTestCertificateChainPem}; diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc index a0cea30a2f8a8..f91734a588b9f 100644 --- a/test/common/secret/sds_api_test.cc +++ b/test/common/secret/sds_api_test.cc @@ -665,7 +665,10 @@ TEST_F(SdsApiTest, DefaultCertificateValidationContextTest) { dynamic_cvc->set_allow_expired_certificate(false); dynamic_cvc->mutable_trusted_ca()->set_filename(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem")); - dynamic_cvc->add_match_subject_alt_names()->set_exact("second san"); + auto* san_matcher = dynamic_cvc->add_match_typed_subject_alt_names(); + san_matcher->mutable_matcher()->set_exact("second san"); + san_matcher->set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS); const std::string dynamic_verify_certificate_spki = "QGJRPdmx/r5EGOFLb2MTiZp2isyC0Whht7iazhzXaCM="; dynamic_cvc->add_verify_certificate_spki(dynamic_verify_certificate_spki); @@ -681,7 +684,10 @@ TEST_F(SdsApiTest, DefaultCertificateValidationContextTest) { envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext default_cvc; default_cvc.set_allow_expired_certificate(true); default_cvc.mutable_trusted_ca()->set_inline_bytes("fake trusted ca"); - default_cvc.add_match_subject_alt_names()->set_exact("first san"); + san_matcher = default_cvc.add_match_typed_subject_alt_names(); + san_matcher->mutable_matcher()->set_exact("first san"); + san_matcher->set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS); default_cvc.add_verify_certificate_hash(default_verify_certificate_hash); envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext merged_cvc = default_cvc; @@ -697,8 +703,12 @@ TEST_F(SdsApiTest, DefaultCertificateValidationContextTest) { cvc_config.caCert()); // Verify that repeated fields are concatenated. EXPECT_EQ(2, cvc_config.subjectAltNameMatchers().size()); - EXPECT_EQ("first san", cvc_config.subjectAltNameMatchers()[0].exact()); - EXPECT_EQ("second san", cvc_config.subjectAltNameMatchers()[1].exact()); + EXPECT_EQ("first san", cvc_config.subjectAltNameMatchers()[0].matcher().exact()); + EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS, + cvc_config.subjectAltNameMatchers()[0].san_type()); + EXPECT_EQ("second san", cvc_config.subjectAltNameMatchers()[1].matcher().exact()); + EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS, + cvc_config.subjectAltNameMatchers()[1].san_type()); // Verify that if dynamic CertificateValidationContext does not set certificate hash list, the new // secret contains hash list from default CertificateValidationContext. EXPECT_EQ(1, cvc_config.verifyCertificateHashList().size()); diff --git a/test/common/secret/secret_manager_impl_test.cc b/test/common/secret/secret_manager_impl_test.cc index a73f143ba8613..1109b7621256b 100644 --- a/test/common/secret/secret_manager_impl_test.cc +++ b/test/common/secret/secret_manager_impl_test.cc @@ -1128,6 +1128,43 @@ name: "abc.com" EnvoyException, "Failed to load private key provider: test"); } +// Verify that using the match_subject_alt_names will result in a typed matcher, one for each of +// DNS, URI, EMAIL and IP_ADDRESS. +// TODO(pradeepcrao): Delete this test once the deprecated field is removed. +TEST_F(SecretManagerImplTest, DeprecatedSanMatcher) { + envoy::extensions::transport_sockets::tls::v3::Secret secret_config; + const std::string yaml = + R"EOF( + name: "abc.com" + validation_context: + trusted_ca: { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" } + allow_expired_certificate: true + match_subject_alt_names: + exact: "example.foo" + )EOF"; + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); + std::unique_ptr secret_manager(new SecretManagerImpl(config_tracker_)); + secret_manager->addStaticSecret(secret_config); + + ASSERT_EQ(secret_manager->findStaticCertificateValidationContextProvider("undefined"), nullptr); + ASSERT_NE(secret_manager->findStaticCertificateValidationContextProvider("abc.com"), nullptr); + Ssl::CertificateValidationContextConfigImpl cvc_config( + *secret_manager->findStaticCertificateValidationContextProvider("abc.com")->secret(), *api_); + EXPECT_EQ(cvc_config.subjectAltNameMatchers().size(), 4); + EXPECT_EQ("example.foo", cvc_config.subjectAltNameMatchers()[0].matcher().exact()); + EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS, + cvc_config.subjectAltNameMatchers()[0].san_type()); + EXPECT_EQ("example.foo", cvc_config.subjectAltNameMatchers()[1].matcher().exact()); + EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::URI, + cvc_config.subjectAltNameMatchers()[1].san_type()); + EXPECT_EQ("example.foo", cvc_config.subjectAltNameMatchers()[2].matcher().exact()); + EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::EMAIL, + cvc_config.subjectAltNameMatchers()[2].san_type()); + EXPECT_EQ("example.foo", cvc_config.subjectAltNameMatchers()[3].matcher().exact()); + EXPECT_EQ(envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::IP_ADDRESS, + cvc_config.subjectAltNameMatchers()[3].san_type()); +} + } // namespace } // namespace Secret } // namespace Envoy diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index b177712ca4cc7..f413d4cfd8a3e 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -3554,9 +3554,13 @@ TEST_F(ClusterInfoImplTest, Http3) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - - exact: localhost - - exact: 127.0.0.1 + match_typed_subject_alt_names: + - matcher: + exact: localhost + san_type: URI + - matcher: + exact: 127.0.0.1 + san_type: IP_ADDRESS )EOF", Network::Address::IpVersion::v4); auto cluster1 = makeCluster(yaml); @@ -3627,9 +3631,13 @@ TEST_F(ClusterInfoImplTest, Http3BadConfig) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - - exact: localhost - - exact: 127.0.0.1 + match_typed_subject_alt_names: + - matcher: + exact: localhost + san_type: URI + - matcher: + exact: 127.0.0.1 + san_type: IP_ADDRESS typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions @@ -3672,9 +3680,13 @@ TEST_F(ClusterInfoImplTest, Http3Auto) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - - exact: localhost - - exact: 127.0.0.1 + match_typed_subject_alt_names: + - matcher: + exact: localhost + san_type: URI + - matcher: + exact: 127.0.0.1 + san_type: IP_ADDRESS )EOF", Network::Address::IpVersion::v4); @@ -3731,9 +3743,13 @@ TEST_F(ClusterInfoImplTest, UseDownstreamHttpProtocolWithoutDowngrade) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - - exact: localhost - - exact: 127.0.0.1 + match_typed_subject_alt_names: + - matcher: + exact: localhost + san_type: URI + - matcher: + exact: 127.0.0.1 + san_type: IP_ADDRESS typed_extension_protocol_options: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions diff --git a/test/config/utility.cc b/test/config/utility.cc index 320fa3957c75c..cfd6fa10e6127 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -1215,8 +1215,8 @@ void ConfigHelper::initializeTls( } } if (!options.san_matchers_.empty()) { - *validation_context->mutable_match_subject_alt_names() = {options.san_matchers_.begin(), - options.san_matchers_.end()}; + *validation_context->mutable_match_typed_subject_alt_names() = {options.san_matchers_.begin(), + options.san_matchers_.end()}; } } diff --git a/test/config/utility.h b/test/config/utility.h index 5ebdc14840192..33fa29e13b69b 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -14,6 +14,7 @@ #include "envoy/config/route/v3/route_components.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" #include "envoy/extensions/transport_sockets/tls/v3/cert.pb.h" +#include "envoy/extensions/transport_sockets/tls/v3/common.pb.h" #include "envoy/extensions/upstreams/http/v3/http_protocol_options.pb.h" #include "envoy/http/codes.h" @@ -80,7 +81,8 @@ class ConfigHelper { } ServerSslOptions& - setSanMatchers(std::vector san_matchers) { + setSanMatchers(std::vector + san_matchers) { san_matchers_ = san_matchers; return *this; } @@ -94,7 +96,8 @@ class ConfigHelper { bool ocsp_staple_required_{false}; bool tlsv1_3_{false}; bool expect_client_ecdsa_cert_{false}; - std::vector san_matchers_{}; + std::vector + san_matchers_{}; }; // Set up basic config, using the specified IpVersion for all connections: listeners, upstream, diff --git a/test/extensions/transport_sockets/tls/cert_validator/BUILD b/test/extensions/transport_sockets/tls/cert_validator/BUILD index 7fd2c97084a17..9d880d51fa05d 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/BUILD +++ b/test/extensions/transport_sockets/tls/cert_validator/BUILD @@ -57,3 +57,16 @@ envoy_cc_test_library( "//test/test_common:utility_lib", ], ) + +envoy_cc_test( + name = "san_matcher_test", + srcs = [ + "san_matcher_test.cc", + ], + deps = [ + "//source/common/protobuf:utility_lib", + "//source/extensions/transport_sockets/tls/cert_validator:cert_validator_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/extensions/transport_sockets/tls/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/transport_sockets/tls/cert_validator/default_validator_test.cc b/test/extensions/transport_sockets/tls/cert_validator/default_validator_test.cc index 036c3679fa1f7..d25eb20c0a037 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/default_validator_test.cc +++ b/test/extensions/transport_sockets/tls/cert_validator/default_validator_test.cc @@ -2,6 +2,7 @@ #include #include "source/extensions/transport_sockets/tls/cert_validator/default_validator.h" +#include "source/extensions/transport_sockets/tls/cert_validator/san_matcher.h" #include "test/extensions/transport_sockets/tls/ssl_test_utility.h" #include "test/test_common/environment.h" @@ -28,22 +29,33 @@ TEST(DefaultCertValidatorTest, TestMatchSubjectAltNameDNSMatched) { bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns_cert.pem")); envoy::type::matcher::v3::StringMatcher matcher; - matcher.MergeFrom(TestUtility::createRegexMatcher(".*.example.com")); - std::vector> - subject_alt_name_matchers; - subject_alt_name_matchers.push_back(Matchers::StringMatcherImpl(matcher)); + matcher.MergeFrom(TestUtility::createRegexMatcher(R"raw([^.]*\.example.com)raw")); + std::vector subject_alt_name_matchers; + subject_alt_name_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_DNS, matcher)}); EXPECT_TRUE(DefaultCertValidator::matchSubjectAltName(cert.get(), subject_alt_name_matchers)); } +TEST(DefaultCertValidatorTest, TestMatchSubjectAltNameIncorrectTypeMatched) { + bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns_cert.pem")); + envoy::type::matcher::v3::StringMatcher matcher; + matcher.MergeFrom(TestUtility::createRegexMatcher(R"raw([^.]*\.example.com)raw")); + std::vector subject_alt_name_matchers; + subject_alt_name_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_URI, matcher)}); + EXPECT_FALSE(DefaultCertValidator::matchSubjectAltName(cert.get(), subject_alt_name_matchers)); +} + TEST(DefaultCertValidatorTest, TestMatchSubjectAltNameWildcardDNSMatched) { bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( "{{ test_rundir " "}}/test/extensions/transport_sockets/tls/test_data/san_multiple_dns_cert.pem")); envoy::type::matcher::v3::StringMatcher matcher; matcher.set_exact("api.example.com"); - std::vector> - subject_alt_name_matchers; - subject_alt_name_matchers.push_back(Matchers::StringMatcherImpl(matcher)); + std::vector subject_alt_name_matchers; + subject_alt_name_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_DNS, matcher)}); EXPECT_TRUE(DefaultCertValidator::matchSubjectAltName(cert.get(), subject_alt_name_matchers)); } @@ -54,9 +66,9 @@ TEST(DefaultCertValidatorTest, TestMultiLevelMatch) { "}}/test/extensions/transport_sockets/tls/test_data/san_multiple_dns_cert.pem")); envoy::type::matcher::v3::StringMatcher matcher; matcher.set_exact("foo.api.example.com"); - std::vector> - subject_alt_name_matchers; - subject_alt_name_matchers.push_back(Matchers::StringMatcherImpl(matcher)); + std::vector subject_alt_name_matchers; + subject_alt_name_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_DNS, matcher)}); EXPECT_FALSE(DefaultCertValidator::matchSubjectAltName(cert.get(), subject_alt_name_matchers)); } @@ -81,10 +93,10 @@ TEST(DefaultCertValidatorTest, TestMatchSubjectAltNameURIMatched) { bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_uri_cert.pem")); envoy::type::matcher::v3::StringMatcher matcher; - matcher.MergeFrom(TestUtility::createRegexMatcher("spiffe://lyft.com/.*-team")); - std::vector> - subject_alt_name_matchers; - subject_alt_name_matchers.push_back(Matchers::StringMatcherImpl(matcher)); + matcher.MergeFrom(TestUtility::createRegexMatcher(R"raw(spiffe://lyft.com/[^/]*-team)raw")); + std::vector subject_alt_name_matchers; + subject_alt_name_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_URI, matcher)}); EXPECT_TRUE(DefaultCertValidator::matchSubjectAltName(cert.get(), subject_alt_name_matchers)); } @@ -100,10 +112,16 @@ TEST(DefaultCertValidatorTest, TestMatchSubjectAltNameNotMatched) { bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns_cert.pem")); envoy::type::matcher::v3::StringMatcher matcher; - matcher.MergeFrom(TestUtility::createRegexMatcher(".*.foo.com")); - std::vector> - subject_alt_name_matchers; - subject_alt_name_matchers.push_back(Matchers::StringMatcherImpl(matcher)); + matcher.MergeFrom(TestUtility::createRegexMatcher(R"raw([^.]*\.example\.net)raw")); + std::vector subject_alt_name_matchers; + subject_alt_name_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_DNS, matcher)}); + subject_alt_name_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_IPADD, matcher)}); + subject_alt_name_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_URI, matcher)}); + subject_alt_name_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_EMAIL, matcher)}); EXPECT_FALSE(DefaultCertValidator::matchSubjectAltName(cert.get(), subject_alt_name_matchers)); } @@ -119,18 +137,18 @@ TEST(DefaultCertValidatorTest, TestCertificateVerificationWithSANMatcher) { bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns_cert.pem")); envoy::type::matcher::v3::StringMatcher matcher; - matcher.MergeFrom(TestUtility::createRegexMatcher(".*.example.com")); - std::vector> san_matchers; - san_matchers.push_back(Matchers::StringMatcherImpl(matcher)); + matcher.MergeFrom(TestUtility::createRegexMatcher(R"raw([^.]*\.example.com)raw")); + std::vector san_matchers; + san_matchers.push_back(SanMatcherPtr{std::make_unique(GEN_DNS, matcher)}); // Verify the certificate with correct SAN regex matcher. EXPECT_EQ(default_validator->verifyCertificate(cert.get(), /*verify_san_list=*/{}, san_matchers), Envoy::Ssl::ClientValidationStatus::Validated); EXPECT_EQ(stats.fail_verify_san_.value(), 0); matcher.MergeFrom(TestUtility::createExactMatcher("hello.example.com")); - std::vector> - invalid_san_matchers; - invalid_san_matchers.push_back(Matchers::StringMatcherImpl(matcher)); + std::vector invalid_san_matchers; + invalid_san_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_DNS, matcher)}); // Verify the certificate with incorrect SAN exact matcher. EXPECT_EQ(default_validator->verifyCertificate(cert.get(), /*verify_san_list=*/{}, invalid_san_matchers), @@ -158,6 +176,17 @@ TEST(DefaultCertValidatorTest, TestCertificateVerificationWithNoValidationContex 0); } +TEST(DefaultCertValidatorTest, NoSanInCert) { + bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/fake_ca_cert.pem")); + envoy::type::matcher::v3::StringMatcher matcher; + matcher.MergeFrom(TestUtility::createRegexMatcher(R"raw([^.]*\.example\.net)raw")); + std::vector subject_alt_name_matchers; + subject_alt_name_matchers.push_back( + SanMatcherPtr{std::make_unique(GEN_DNS, matcher)}); + EXPECT_FALSE(DefaultCertValidator::matchSubjectAltName(cert.get(), subject_alt_name_matchers)); +} + } // namespace Tls } // namespace TransportSockets } // namespace Extensions diff --git a/test/extensions/transport_sockets/tls/cert_validator/san_matcher_test.cc b/test/extensions/transport_sockets/tls/cert_validator/san_matcher_test.cc new file mode 100644 index 0000000000000..b7729de8d41ea --- /dev/null +++ b/test/extensions/transport_sockets/tls/cert_validator/san_matcher_test.cc @@ -0,0 +1,57 @@ +#include "envoy/extensions/transport_sockets/tls/v3/common.pb.validate.h" + +#include "source/common/protobuf/message_validator_impl.h" +#include "source/common/protobuf/utility.h" +#include "source/extensions/transport_sockets/tls/cert_validator/san_matcher.h" + +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { + +// Verify that we get a valid string san matcher for all valid san types. +TEST(SanMatcherConfigTest, TestValidSanType) { + // Iterate over all san type enums. + for (envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::SanType san_type = + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::SanType_MIN; + san_type <= + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::SanType_MAX; + san_type = static_cast< + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::SanType>( + static_cast(san_type + 1))) { + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher san_matcher; + san_matcher.mutable_matcher()->set_exact("foo.example"); + san_matcher.set_san_type(san_type); + if (san_type == envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher:: + SAN_TYPE_UNSPECIFIED) { + continue; + } else { + const SanMatcherPtr matcher = createStringSanMatcher(san_matcher); + EXPECT_NE(matcher.get(), nullptr); + // Verify that the message is valid. + TestUtility::validate(san_matcher); + } + } +} + +TEST(SanMatcherConfigTest, UnspecifiedSanType) { + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher san_matcher; + san_matcher.mutable_matcher()->set_exact("foo.example"); + // Do not set san_type + EXPECT_THROW_WITH_REGEX(TestUtility::validate(san_matcher), EnvoyException, + "Proto constraint validation failed"); + san_matcher.set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::SAN_TYPE_UNSPECIFIED); + EXPECT_THROW_WITH_REGEX(TestUtility::validate(san_matcher), EnvoyException, + "Proto constraint validation failed"); +} + +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.cc b/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.cc index 47acf98c4f533..fe47ef14f6e3c 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.cc +++ b/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.cc @@ -53,6 +53,26 @@ void SslSPIFFECertValidatorIntegrationTest::checkVerifyErrorCouter(uint64_t valu counter->reset(); } +void SslSPIFFECertValidatorIntegrationTest::addStringMatcher( + const envoy::type::matcher::v3::StringMatcher& matcher) { + san_matchers_.emplace_back(); + *san_matchers_.back().mutable_matcher() = matcher; + san_matchers_.back().set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS); + san_matchers_.emplace_back(); + *san_matchers_.back().mutable_matcher() = matcher; + san_matchers_.back().set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::URI); + san_matchers_.emplace_back(); + *san_matchers_.back().mutable_matcher() = matcher; + san_matchers_.back().set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::EMAIL); + san_matchers_.emplace_back(); + *san_matchers_.back().mutable_matcher() = matcher; + san_matchers_.back().set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::IP_ADDRESS); +} + INSTANTIATE_TEST_SUITE_P( IpVersionsClientVersions, SslSPIFFECertValidatorIntegrationTest, testing::Combine( @@ -124,7 +144,7 @@ name: envoy.tls.cert_validator.spiffe envoy::type::matcher::v3::StringMatcher matcher; matcher.set_prefix("spiffe://lyft.com/"); - san_matchers_ = {matcher}; + addStringMatcher(matcher); ConnectionCreationFunction creator = [&]() -> Network::ClientConnectionPtr { return makeSslClientConnection({}); @@ -152,7 +172,7 @@ name: envoy.tls.cert_validator.spiffe matcher.set_prefix("spiffe://example.com/"); // The cert has "DNS.1 = lyft.com" but SPIFFE validator must ignore SAN types other than URI. matcher.set_prefix("www.lyft.com"); - san_matchers_ = {matcher}; + addStringMatcher(matcher); initialize(); auto conn = makeSslClientConnection({}); if (tls_version_ == envoy::extensions::transport_sockets::tls::v3::TlsParameters::TLSv1_2) { @@ -223,8 +243,8 @@ name: envoy.tls.cert_validator.spiffe checkVerifyErrorCouter(1); } -// clientcert.pem's san is "spiffe://lyft.com/frontend-team" but the corresponding trust bundle does -// not match with the client cert. So this should also be rejected. +// clientcert.pem's san is "spiffe://lyft.com/frontend-team" but the corresponding trust bundle +// does not match with the client cert. So this should also be rejected. TEST_P(SslSPIFFECertValidatorIntegrationTest, ServerRsaSPIFFEValidatorRejected2) { auto typed_conf = new envoy::config::core::v3::TypedExtensionConfig(); TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( diff --git a/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.h b/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.h index b1e57169ea18f..01d08f5a811d4 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.h +++ b/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_integration_test.h @@ -39,10 +39,11 @@ class SslSPIFFECertValidatorIntegrationTest } protected: + void addStringMatcher(envoy::type::matcher::v3::StringMatcher const& matcher); bool allow_expired_cert_{}; envoy::config::core::v3::TypedExtensionConfig* custom_validator_config_{nullptr}; std::unique_ptr context_manager_; - std::vector san_matchers_; + std::vector san_matchers_; const envoy::extensions::transport_sockets::tls::v3::TlsParameters::TlsProtocol tls_version_{ std::get<1>(GetParam())}; }; diff --git a/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_test.cc b/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_test.cc index 07206e156a72b..3be9706e76be1 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_test.cc +++ b/test/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator_test.cc @@ -64,13 +64,34 @@ class TestSPIFFEValidator : public testing::Test { // Setter. void setAllowExpiredCertificate(bool val) { allow_expired_certificate_ = val; } void setSanMatchers(std::vector san_matchers) { - san_matchers_ = san_matchers; + san_matchers_.clear(); + for (auto& matcher : san_matchers) { + san_matchers_.emplace_back(); + san_matchers_.back().set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS); + *san_matchers_.back().mutable_matcher() = matcher; + + san_matchers_.emplace_back(); + san_matchers_.back().set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::URI); + *san_matchers_.back().mutable_matcher() = matcher; + + san_matchers_.emplace_back(); + san_matchers_.back().set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::EMAIL); + *san_matchers_.back().mutable_matcher() = matcher; + + san_matchers_.emplace_back(); + san_matchers_.back().set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::IP_ADDRESS); + *san_matchers_.back().mutable_matcher() = matcher; + } }; private: bool allow_expired_certificate_{false}; TestCertificateValidationContextConfigPtr config_; - std::vector san_matchers_{}; + std::vector san_matchers_{}; Stats::TestUtil::TestStore store_; SslStats stats_; Event::TestRealTimeSystem time_system_; @@ -193,7 +214,8 @@ TEST_F(TestSPIFFEValidator, TestGetTrustBundleStore) { // Non-SPIFFE SAN cert = readCertFromFile(TestEnvironment::substitute( - "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/non_spiffe_san_cert.pem")); + "{{ test_rundir " + "}}/test/extensions/transport_sockets/tls/test_data/non_spiffe_san_cert.pem")); EXPECT_FALSE(validator().getTrustBundleStore(cert.get())); // SPIFFE SAN diff --git a/test/extensions/transport_sockets/tls/cert_validator/test_common.h b/test/extensions/transport_sockets/tls/cert_validator/test_common.h index 11d4022cf0436..0eb5066e1c760 100644 --- a/test/extensions/transport_sockets/tls/cert_validator/test_common.h +++ b/test/extensions/transport_sockets/tls/cert_validator/test_common.h @@ -33,7 +33,8 @@ class TestCertificateValidationContextConfig public: TestCertificateValidationContextConfig( envoy::config::core::v3::TypedExtensionConfig config, bool allow_expired_certificate = false, - std::vector san_matchers = {}) + std::vector + san_matchers = {}) : allow_expired_certificate_(allow_expired_certificate), api_(Api::createApiForTest()), custom_validator_config_(config), san_matchers_(san_matchers){}; TestCertificateValidationContextConfig() @@ -47,7 +48,7 @@ class TestCertificateValidationContextConfig const std::string& certificateRevocationListPath() const final { CONSTRUCT_ON_FIRST_USE(std::string, ""); } - const std::vector& + const std::vector& subjectAltNameMatchers() const override { return san_matchers_; } @@ -78,7 +79,8 @@ class TestCertificateValidationContextConfig bool allow_expired_certificate_{false}; Api::ApiPtr api_; const absl::optional custom_validator_config_; - const std::vector san_matchers_{}; + const std::vector + san_matchers_{}; }; } // namespace Tls diff --git a/test/extensions/transport_sockets/tls/context_impl_test.cc b/test/extensions/transport_sockets/tls/context_impl_test.cc index 2e6c67a1da178..f4568860b223c 100644 --- a/test/extensions/transport_sockets/tls/context_impl_test.cc +++ b/test/extensions/transport_sockets/tls/context_impl_test.cc @@ -923,8 +923,10 @@ TEST_F(SslServerContextImplTicketTest, VerifySanWithNoCA) { private_key: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns_key.pem" validation_context: - match_subject_alt_names: - exact : "spiffe://lyft.com/testclient" + match_typed_subject_alt_names: + - san_type: URI + matcher: + exact: "spiffe://lyft.com/testclient" )EOF"; EXPECT_THROW_WITH_MESSAGE(loadConfigYaml(yaml), EnvoyException, "SAN-based verification of peer certificates without trusted CA " @@ -1907,6 +1909,35 @@ TEST_F(ServerContextConfigImplTest, PrivateKeyMethodLoadFailureBothKeyAndMethod) "Certificate configuration can't have both private_key and private_key_provider"); } +// Test that we don't allow specification of both typed and untyped matchers for +// sans. +TEST_F(ServerContextConfigImplTest, DeprecatedSanMatcher) { + envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; + NiceMock context_manager; + NiceMock private_key_method_manager; + auto private_key_method_provider_ptr = + std::make_shared>(); + const std::string yaml = + R"EOF( + common_tls_context: + validation_context: + trusted_ca: { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" } + allow_expired_certificate: true + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: "foo.example" + match_subject_alt_names: + exact: "foo.example" + )EOF"; + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), tls_context); + + EXPECT_THROW_WITH_MESSAGE( + ServerContextConfigImpl server_context_config(tls_context, factory_context_), EnvoyException, + "SAN-based verification using both match_typed_subject_alt_names and " + "the deprecated match_subject_alt_names is not allowed"); +} + TEST_F(ServerContextConfigImplTest, Pkcs12LoadFailureBothPkcs12AndMethod) { envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext tls_context; NiceMock context_manager; diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 751d9e5459b9d..af9288bbea316 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -1089,8 +1089,10 @@ TEST_P(SslSocketTest, GetUriWithUriSan) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - exact: "spiffe://lyft.com/test-team" + match_typed_subject_alt_names: + - san_type: URI + matcher: + exact: "spiffe://lyft.com/test-team" )EOF"; TestUtilOptions test_options(client_ctx_yaml, server_ctx_yaml, true, GetParam()); @@ -1105,8 +1107,10 @@ TEST_P(SslSocketTest, Ipv4San) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/config/integration/certs/upstreamcacert.pem" - match_subject_alt_names: - exact: "127.0.0.1" + match_typed_subject_alt_names: + - san_type: IP_ADDRESS + matcher: + exact: "127.0.0.1" )EOF"; const std::string server_ctx_yaml = R"EOF( @@ -1129,8 +1133,10 @@ TEST_P(SslSocketTest, Ipv6San) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/config/integration/certs/upstreamcacert.pem" - match_subject_alt_names: - exact: "::1" + match_typed_subject_alt_names: + - san_type: IP_ADDRESS + matcher: + exact: "::1" )EOF"; const std::string server_ctx_yaml = R"EOF( @@ -1533,8 +1539,10 @@ TEST_P(SslSocketTest, FailedClientAuthSanVerificationNoClientCert) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - exact: "example.com" + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: "example.com" )EOF"; TestUtilOptions test_options(client_ctx_yaml, server_ctx_yaml, false, GetParam()); @@ -1561,8 +1569,10 @@ TEST_P(SslSocketTest, FailedClientAuthSanVerification) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - exact: "example.com" + match_typed_subject_alt_names: + - san_type: DNS + matcher: + exact: "example.com" )EOF"; TestUtilOptions test_options(client_ctx_yaml, server_ctx_yaml, false, GetParam()); diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index f97ad753fe527..a85242d72d1bf 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -145,7 +145,10 @@ void AdsIntegrationTest::initializeAds(const bool rate_limiting) { auto* validation_context = context.mutable_common_tls_context()->mutable_validation_context(); validation_context->mutable_trusted_ca()->set_filename( TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); - validation_context->add_match_subject_alt_names()->set_suffix("lyft.com"); + auto* san_matcher = validation_context->add_match_typed_subject_alt_names(); + san_matcher->mutable_matcher()->set_suffix("lyft.com"); + san_matcher->set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS); if (clientType() == Grpc::ClientType::GoogleGrpc) { auto* google_grpc = grpc_service->mutable_google_grpc(); auto* ssl_creds = google_grpc->mutable_channel_credentials()->mutable_ssl_credentials(); diff --git a/test/integration/ssl_utility.cc b/test/integration/ssl_utility.cc index 36b93b6f81bca..c629d6f2e3c0a 100644 --- a/test/integration/ssl_utility.cc +++ b/test/integration/ssl_utility.cc @@ -66,8 +66,23 @@ void initializeUpstreamTlsContextConfig( common_context->add_alpn_protocols(Http::Utility::AlpnNames::get().Http3); } if (!options.san_.empty()) { - common_context->mutable_validation_context()->add_match_subject_alt_names()->set_exact( - options.san_); + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher* matcher = + common_context->mutable_validation_context()->add_match_typed_subject_alt_names(); + matcher->mutable_matcher()->set_exact(options.san_); + matcher->set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS); + matcher = common_context->mutable_validation_context()->add_match_typed_subject_alt_names(); + matcher->mutable_matcher()->set_exact(options.san_); + matcher->set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::URI); + matcher = common_context->mutable_validation_context()->add_match_typed_subject_alt_names(); + matcher->mutable_matcher()->set_exact(options.san_); + matcher->set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::EMAIL); + matcher = common_context->mutable_validation_context()->add_match_typed_subject_alt_names(); + matcher->mutable_matcher()->set_exact(options.san_); + matcher->set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::IP_ADDRESS); } for (const std::string& cipher_suite : options.cipher_suites_) { common_context->mutable_tls_params()->add_cipher_suites(cipher_suite); diff --git a/test/integration/xfcc_integration_test.cc b/test/integration/xfcc_integration_test.cc index 19186660db21a..0ce4d4a94213c 100644 --- a/test/integration/xfcc_integration_test.cc +++ b/test/integration/xfcc_integration_test.cc @@ -6,6 +6,7 @@ #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" #include "envoy/extensions/transport_sockets/tls/v3/cert.pb.h" +#include "envoy/extensions/transport_sockets/tls/v3/common.pb.h" #include "envoy/stats/scope.h" #include "source/common/event/dispatcher_impl.h" @@ -45,10 +46,16 @@ Network::TransportSocketFactoryPtr XfccIntegrationTest::createClientSslContext(b validation_context: trusted_ca: filename: {{ test_rundir }}/test/config/integration/certs/cacert.pem - match_subject_alt_names: - exact: "spiffe://lyft.com/backend-team" - exact: "lyft.com" - exact: "www.lyft.com" + match_typed_subject_alt_names: + - san_type: URI + matcher: + exact: "spiffe://lyft.com/backend-team" + - san_type: DNS + matcher: + exact: "lyft.com" + - san_type: DNS + matcher: + exact: "www.lyft.com" )EOF"; const std::string yaml_mtls = R"EOF( @@ -56,10 +63,16 @@ Network::TransportSocketFactoryPtr XfccIntegrationTest::createClientSslContext(b validation_context: trusted_ca: filename: {{ test_rundir }}/test/config/integration/certs/cacert.pem - match_subject_alt_names: - exact: "spiffe://lyft.com/backend-team" - exact: "lyft.com" - exact: "www.lyft.com" + match_typed_subject_alt_names: + - san_type: URI + matcher: + exact: "spiffe://lyft.com/backend-team" + - san_type: DNS + matcher: + exact: "lyft.com" + - san_type: DNS + matcher: + exact: "www.lyft.com" tls_certificates: certificate_chain: filename: {{ test_rundir }}/test/config/integration/certs/clientcert.pem @@ -135,7 +148,10 @@ void XfccIntegrationTest::initialize() { auto* validation_context = context.mutable_common_tls_context()->mutable_validation_context(); validation_context->mutable_trusted_ca()->set_filename( TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); - validation_context->add_match_subject_alt_names()->set_suffix("lyft.com"); + auto* san_matcher = validation_context->add_match_typed_subject_alt_names(); + san_matcher->mutable_matcher()->set_suffix("lyft.com"); + san_matcher->set_san_type( + envoy::extensions::transport_sockets::tls::v3::SubjectAltNameMatcher::DNS); transport_socket->set_name("envoy.transport_sockets.tls"); transport_socket->mutable_typed_config()->PackFrom(context); }); diff --git a/test/mocks/ssl/mocks.h b/test/mocks/ssl/mocks.h index 130a78b83c61e..104242ce0c926 100644 --- a/test/mocks/ssl/mocks.h +++ b/test/mocks/ssl/mocks.h @@ -159,8 +159,9 @@ class MockCertificateValidationContextConfig : public CertificateValidationConte MOCK_METHOD(const std::string&, caCertPath, (), (const)); MOCK_METHOD(const std::string&, certificateRevocationList, (), (const)); MOCK_METHOD(const std::string&, certificateRevocationListPath, (), (const)); - MOCK_METHOD(const std::vector&, subjectAltNameMatchers, - (), (const)); + MOCK_METHOD( + const std::vector&, + subjectAltNameMatchers, (), (const)); MOCK_METHOD(const std::vector&, verifyCertificateHashList, (), (const)); MOCK_METHOD(const std::vector&, verifyCertificateSpkiList, (), (const)); MOCK_METHOD(bool, allowExpiredCertificate, (), (const)); diff --git a/test/per_file_coverage.sh b/test/per_file_coverage.sh index 184a5485da260..93ba06fd0c3f4 100755 --- a/test/per_file_coverage.sh +++ b/test/per_file_coverage.sh @@ -71,7 +71,7 @@ declare -a KNOWN_LOW_COVERAGE=( "source/extensions/tracers/zipkin:96.1" "source/extensions/transport_sockets:95.3" "source/extensions/transport_sockets/tls:94.5" -"source/extensions/transport_sockets/tls/cert_validator:95.8" +"source/extensions/transport_sockets/tls/cert_validator:95.7" "source/extensions/transport_sockets/tls/ocsp:96.5" "source/extensions/transport_sockets/tls/private_key:77.8" "source/extensions/wasm_runtime/wamr:0.0" # Not enabled in coverage build diff --git a/test/server/listener_manager_impl_quic_only_test.cc b/test/server/listener_manager_impl_quic_only_test.cc index 84e1a3a200c95..f505b467c99dc 100644 --- a/test/server/listener_manager_impl_quic_only_test.cc +++ b/test/server/listener_manager_impl_quic_only_test.cc @@ -60,9 +60,13 @@ TEST_F(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryAndSslContext) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - - exact: localhost - - exact: 127.0.0.1 + match_typed_subject_alt_names: + - matcher: + exact: localhost + san_type: URI + - matcher: + exact: 127.0.0.1 + san_type: IP_ADDRESS udp_listener_config: quic_options: {} )EOF", @@ -161,9 +165,13 @@ TEST_F(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryWithWrongTransportSoc validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - - exact: localhost - - exact: 127.0.0.1 + match_typed_subject_alt_names: + - matcher: + exact: localhost + san_type: URI + - matcher: + exact: 127.0.0.1 + san_type: IP_ADDRESS udp_listener_config: quic_options: {} )EOF", @@ -205,9 +213,13 @@ TEST_F(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryWithWrongCodec) { validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - - exact: localhost - - exact: 127.0.0.1 + match_typed_subject_alt_names: + - matcher: + exact: localhost + san_type: URI + - matcher: + exact: 127.0.0.1 + san_type: IP_ADDRESS udp_listener_config: quic_options: {} )EOF", @@ -259,9 +271,13 @@ TEST_F(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryWithConnectionBalence validation_context: trusted_ca: filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" - match_subject_alt_names: - - exact: localhost - - exact: 127.0.0.1 + match_typed_subject_alt_names: + - matcher: + exact: localhost + san_type: URI + - matcher: + exact: 127.0.0.1 + san_type: IP_ADDRESS udp_listener_config: quic_options: {} connection_balance_config: From e903a6e3adcc21618cb2315c58699b3f0d04c42b Mon Sep 17 00:00:00 2001 From: Weston Carlson Date: Wed, 24 Nov 2021 09:08:48 -0600 Subject: [PATCH 13/51] Listener: Add global conn limit opt out. (#18876) Signed-off-by: Weston Carlson --- api/envoy/config/bootstrap/v3/bootstrap.proto | 6 +- api/envoy/config/listener/v3/listener.proto | 6 +- ci/README.md | 2 +- .../overload_manager/overload_manager.rst | 9 ++ docs/root/version_history/current.rst | 1 + envoy/event/dispatcher.h | 6 +- envoy/network/listener.h | 6 + envoy/server/configuration.h | 6 + source/common/event/dispatcher_impl.cc | 7 +- source/common/event/dispatcher_impl.h | 4 +- source/common/network/tcp_listener_impl.cc | 9 +- source/common/network/tcp_listener_impl.h | 6 +- source/server/active_tcp_listener.cc | 2 +- source/server/admin/admin.cc | 7 +- source/server/admin/admin.h | 8 +- source/server/config_validation/dispatcher.cc | 3 +- source/server/config_validation/dispatcher.h | 2 +- source/server/configuration_impl.cc | 1 + source/server/configuration_impl.h | 2 + source/server/listener_impl.cc | 2 + source/server/listener_impl.h | 2 + source/server/server.cc | 3 +- test/common/http/codec_client_test.cc | 3 +- test/common/network/connection_impl_test.cc | 6 +- test/common/network/listener_impl_test.cc | 84 +++++++++-- .../proxy_protocol_regression_test.cc | 1 + .../proxy_protocol/proxy_protocol_test.cc | 2 + .../dns_resolver/cares/dns_impl_test.cc | 2 +- .../transport_sockets/tls/ssl_socket_test.cc | 33 +++-- test/integration/cx_limit_integration_test.cc | 137 +++++++++++++++++- test/integration/fake_upstream.h | 1 + test/mocks/event/mocks.h | 9 +- test/mocks/event/wrapped_dispatcher.h | 6 +- test/mocks/network/mocks.h | 1 + test/server/admin/admin_instance.cc | 2 +- test/server/admin/admin_test.cc | 4 +- test/server/admin/profiling_handler_test.cc | 2 +- test/server/connection_handler_test.cc | 16 +- 38 files changed, 324 insertions(+), 85 deletions(-) diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto index 0cb494e163235..d056ba7469550 100644 --- a/api/envoy/config/bootstrap/v3/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3/bootstrap.proto @@ -329,7 +329,7 @@ message Bootstrap { // Administration interface :ref:`operations documentation // `. -// [#next-free-field: 6] +// [#next-free-field: 7] message Admin { option (udpa.annotations.versioning).previous_message_type = "envoy.config.bootstrap.v2.Admin"; @@ -355,6 +355,10 @@ message Admin { // Additional socket options that may not be present in Envoy source code or // precompiled binaries. repeated core.v3.SocketOption socket_options = 4; + + // Indicates whether :ref:`global_downstream_max_connections ` + // should apply to the admin interface or not. + bool ignore_global_conn_limit = 6; } // Cluster manager :ref:`architecture overview `. diff --git a/api/envoy/config/listener/v3/listener.proto b/api/envoy/config/listener/v3/listener.proto index f065ff67160b3..a207875cae237 100644 --- a/api/envoy/config/listener/v3/listener.proto +++ b/api/envoy/config/listener/v3/listener.proto @@ -35,7 +35,7 @@ message ListenerCollection { repeated xds.core.v3.CollectionEntry entries = 1; } -// [#next-free-field: 31] +// [#next-free-field: 32] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Listener"; @@ -318,4 +318,8 @@ message Listener { // Enable MPTCP (multi-path TCP) on this listener. Clients will be allowed to establish // MPTCP connections. Non-MPTCP clients will fall back to regular TCP. bool enable_mptcp = 30; + + // Whether the listener should limit connections based upon the value of + // :ref:`global_downstream_max_connections `. + bool ignore_global_conn_limit = 31; } diff --git a/ci/README.md b/ci/README.md index 0561facedf73c..fd4a3b2d63532 100644 --- a/ci/README.md +++ b/ci/README.md @@ -135,7 +135,7 @@ The `./ci/run_envoy_docker.sh './ci/do_ci.sh '` targets are: * `bazel.clang_tidy ` — build and run clang-tidy specified source files, if no files specified, runs against the diff with the last GitHub commit. * `check_format`— run `clang-format` and `buildifier` on entire source tree. * `fix_format`— run and enforce `clang-format` and `buildifier` on entire source tree. -* `check_spelling_pedantic`— run `aspell` on C++ and proto comments. +* `format_pre`— run validation and linting tools. * `docs`— build documentation tree in `generated/docs`. ## On Windows diff --git a/docs/root/configuration/operations/overload_manager/overload_manager.rst b/docs/root/configuration/operations/overload_manager/overload_manager.rst index 7dacd28b323b0..48f5aa9499da3 100644 --- a/docs/root/configuration/operations/overload_manager/overload_manager.rst +++ b/docs/root/configuration/operations/overload_manager/overload_manager.rst @@ -143,6 +143,8 @@ Note in the example that the minimum idle time is specified as an absolute durat would be computed based on the maximum (specified elsewhere). So if ``idle_timeout`` is again 600 seconds, then the minimum timer value would be :math:`10\% \cdot 600s = 60s`. +.. _config_overload_manager_limiting_connections: + Limiting Active Connections --------------------------- @@ -155,6 +157,13 @@ If the value is unspecified, there is no global limit on the number of active do and Envoy will emit a warning indicating this at startup. To disable the warning without setting a limit on the number of active downstream connections, the runtime value may be set to a very large limit (~2e9). +Listeners can opt out of this global connection limit by setting +:ref:`Listener.ignore_global_conn_limit ` +to true. Similarly, you can opt out the admin listener by setting +:ref:`Admin.ignore_global_conn_limit `. +You may want to opt out a listener to be able to probe Envoy or collect stats while it is otherwise at its +connection limit. Note that connections to listeners that opt out are still tracked and count towards the +global limit. If it is desired to only limit the number of downstream connections for a particular listener, per-listener limits can be set via the :ref:`listener configuration `. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 1937c5298b410..93acb4131cdd7 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -69,6 +69,7 @@ New Features * http: added timing information about upstream connection and encryption establishment to stream info. These can currently be accessed via custom access loggers. * listener: added API for extensions to access :ref:`typed_filter_metadata ` configured in the listener's :ref:`metadata ` field. * listener: added support for :ref:`MPTCP ` (multipath TCP). +* listener: added support for opting out listeners from the globally set downstream connection limit via :ref:`ignore_global_conn_limit `. * oauth filter: added :ref:`cookie_names ` to allow overriding (default) cookie names (``BearerToken``, ``OauthHMAC``, and ``OauthExpires``) set by the filter. * oauth filter: setting IdToken and RefreshToken cookies if they are provided by Identity provider along with AccessToken. * router: added support for the :ref:`config_http_conn_man_headers_x-forwarded-host` header. diff --git a/envoy/event/dispatcher.h b/envoy/event/dispatcher.h index 94cb73e5c4fbc..86fd2855a1512 100644 --- a/envoy/event/dispatcher.h +++ b/envoy/event/dispatcher.h @@ -228,11 +228,13 @@ class Dispatcher : public DispatcherBase, public ScopeTracker { * @param socket supplies the socket to listen on. * @param cb supplies the callbacks to invoke for listener events. * @param bind_to_port controls whether the listener binds to a transport port or not. + * @param ignore_global_conn_limit controls whether the listener is limited by the global + * connection limit. * @return Network::ListenerPtr a new listener that is owned by the caller. */ virtual Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::TcpListenerCallbacks& cb, - bool bind_to_port) PURE; + Network::TcpListenerCallbacks& cb, bool bind_to_port, + bool ignore_global_conn_limit) PURE; /** * Creates a logical udp listener on a specific port. diff --git a/envoy/network/listener.h b/envoy/network/listener.h index 9969c4198fa93..fac995eb3ed6f 100644 --- a/envoy/network/listener.h +++ b/envoy/network/listener.h @@ -230,6 +230,12 @@ class ListenerConfig { * @return init manager of the listener. */ virtual Init::Manager& initManager() PURE; + + /** + * @return bool whether the listener should avoid blocking connections based on the globally set + * limit. + */ + virtual bool ignoreGlobalConnLimit() const PURE; }; /** diff --git a/envoy/server/configuration.h b/envoy/server/configuration.h index b43da186eabcd..32c7e3f661b17 100644 --- a/envoy/server/configuration.h +++ b/envoy/server/configuration.h @@ -141,6 +141,12 @@ class Admin { * @return Network::Address::OptionsSharedPtr the list of listener socket options. */ virtual Network::Socket::OptionsSharedPtr socketOptions() PURE; + + /** + * @return bool whether the listener should avoid blocking connections based on the globally set + * limit. + */ + virtual bool ignoreGlobalConnLimit() const PURE; }; /** diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 1f28e9a24fb85..a94b537ff45e7 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -171,10 +171,11 @@ Filesystem::WatcherPtr DispatcherImpl::createFilesystemWatcher() { Network::ListenerPtr DispatcherImpl::createListener(Network::SocketSharedPtr&& socket, Network::TcpListenerCallbacks& cb, - bool bind_to_port) { + bool bind_to_port, + bool ignore_global_conn_limit) { ASSERT(isThreadSafe()); - return std::make_unique(*this, api_.randomGenerator(), - std::move(socket), cb, bind_to_port); + return std::make_unique( + *this, api_.randomGenerator(), std::move(socket), cb, bind_to_port, ignore_global_conn_limit); } Network::UdpListenerPtr diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index 513ebc9d1b2b3..068638b727252 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -70,8 +70,8 @@ class DispatcherImpl : Logger::Loggable, uint32_t events) override; Filesystem::WatcherPtr createFilesystemWatcher() override; Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::TcpListenerCallbacks& cb, - bool bind_to_port) override; + Network::TcpListenerCallbacks& cb, bool bind_to_port, + bool ignore_global_conn_limit) override; Network::UdpListenerPtr createUdpListener(Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb, const envoy::config::core::v3::UdpSocketConfig& config) override; diff --git a/source/common/network/tcp_listener_impl.cc b/source/common/network/tcp_listener_impl.cc index a3f79f72df6b6..b73f26ab4df0e 100644 --- a/source/common/network/tcp_listener_impl.cc +++ b/source/common/network/tcp_listener_impl.cc @@ -20,11 +20,11 @@ namespace Network { const absl::string_view TcpListenerImpl::GlobalMaxCxRuntimeKey = "overload.global_downstream_max_connections"; -bool TcpListenerImpl::rejectCxOverGlobalLimit() { +bool TcpListenerImpl::rejectCxOverGlobalLimit() const { // Enforce the global connection limit if necessary, immediately closing the accepted connection. Runtime::Loader* runtime = Runtime::LoaderSingleton::getExisting(); - if (runtime == nullptr) { + if (ignore_global_conn_limit_ || runtime == nullptr) { // The runtime singleton won't exist in most unit tests that do not need global downstream limit // enforcement. Therefore, there is no need to enforce limits if the singleton doesn't exist. // TODO(tonya11en): Revisit this once runtime is made globally available. @@ -98,9 +98,10 @@ void TcpListenerImpl::onSocketEvent(short flags) { TcpListenerImpl::TcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random, SocketSharedPtr socket, TcpListenerCallbacks& cb, - bool bind_to_port) + bool bind_to_port, bool ignore_global_conn_limit) : BaseListenerImpl(dispatcher, std::move(socket)), cb_(cb), random_(random), - bind_to_port_(bind_to_port), reject_fraction_(0.0) { + bind_to_port_(bind_to_port), reject_fraction_(0.0), + ignore_global_conn_limit_(ignore_global_conn_limit) { if (bind_to_port) { // Although onSocketEvent drains to completion, use level triggered mode to avoid potential // loss of the trigger due to transient accept errors. diff --git a/source/common/network/tcp_listener_impl.h b/source/common/network/tcp_listener_impl.h index 27ecfeec949b6..f9978b73543ef 100644 --- a/source/common/network/tcp_listener_impl.h +++ b/source/common/network/tcp_listener_impl.h @@ -17,7 +17,8 @@ namespace Network { class TcpListenerImpl : public BaseListenerImpl { public: TcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random, - SocketSharedPtr socket, TcpListenerCallbacks& cb, bool bind_to_port); + SocketSharedPtr socket, TcpListenerCallbacks& cb, bool bind_to_port, + bool ignore_global_conn_limit); ~TcpListenerImpl() override { if (bind_to_port_) { socket_->ioHandle().resetFileEvents(); @@ -37,11 +38,12 @@ class TcpListenerImpl : public BaseListenerImpl { // Returns true if global connection limit has been reached and the accepted socket should be // rejected/closed. If the accepted socket is to be admitted, false is returned. - static bool rejectCxOverGlobalLimit(); + bool rejectCxOverGlobalLimit() const; Random::RandomGenerator& random_; bool bind_to_port_; UnitFloat reject_fraction_; + const bool ignore_global_conn_limit_; }; } // namespace Network diff --git a/source/server/active_tcp_listener.cc b/source/server/active_tcp_listener.cc index 3e0f850c9b7a6..763a9170d4b2e 100644 --- a/source/server/active_tcp_listener.cc +++ b/source/server/active_tcp_listener.cc @@ -17,7 +17,7 @@ ActiveTcpListener::ActiveTcpListener(Network::TcpConnectionHandler& parent, : OwnedActiveStreamListenerBase(parent, parent.dispatcher(), parent.dispatcher().createListener( config.listenSocketFactory().getListenSocket(worker_index), - *this, config.bindToPort()), + *this, config.bindToPort(), config.ignoreGlobalConnLimit()), config), tcp_conn_handler_(parent) { config.connectionBalancer().registerHandler(*this); diff --git a/source/server/admin/admin.cc b/source/server/admin/admin.cc index 74c17040b261f..0039d7cf4567b 100644 --- a/source/server/admin/admin.cc +++ b/source/server/admin/admin.cc @@ -109,7 +109,6 @@ const char AdminHtmlEnd[] = R"( )"; - } // namespace ConfigTracker& AdminImpl::getConfigTracker() { return config_tracker_; } @@ -144,7 +143,8 @@ void AdminImpl::startHttpListener(const std::list& } } -AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server) +AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server, + bool ignore_global_conn_limit) : server_(server), request_id_extension_(Extensions::RequestId::UUIDRequestIDExtension::defaultInstance( server_.api().randomGenerator())), @@ -221,7 +221,8 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server) }, date_provider_(server.dispatcher().timeSource()), admin_filter_chain_(std::make_shared()), - local_reply_(LocalReply::Factory::createDefault()) {} + local_reply_(LocalReply::Factory::createDefault()), + ignore_global_conn_limit_(ignore_global_conn_limit) {} Http::ServerConnectionPtr AdminImpl::createCodec(Network::Connection& connection, const Buffer::Instance& data, diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index abac710da8698..7ee70f069d01f 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -68,7 +68,8 @@ class AdminImpl : public Admin, public Http::ConnectionManagerConfig, Logger::Loggable { public: - AdminImpl(const std::string& profile_path, Server::Instance& server); + AdminImpl(const std::string& profile_path, Server::Instance& server, + bool ignore_global_conn_limit); Http::Code runCallback(absl::string_view path_and_query, Http::ResponseHeaderMap& response_headers, Buffer::Instance& response, @@ -340,7 +341,7 @@ class AdminImpl : public Admin, AdminListener(AdminImpl& parent, Stats::ScopePtr&& listener_scope) : parent_(parent), name_("admin"), scope_(std::move(listener_scope)), stats_(Http::ConnectionManagerImpl::generateListenerStats("http.admin.", *scope_)), - init_manager_(nullptr) {} + init_manager_(nullptr), ignore_global_conn_limit_(parent.ignore_global_conn_limit_) {} // Network::ListenerConfig Network::FilterChainManager& filterChainManager() override { return parent_; } @@ -372,6 +373,7 @@ class AdminImpl : public Admin, } uint32_t tcpBacklogSize() const override { return ENVOY_TCP_BACKLOG_SIZE; } Init::Manager& initManager() override { return *init_manager_; } + bool ignoreGlobalConnLimit() const override { return ignore_global_conn_limit_; } AdminImpl& parent_; const std::string name_; @@ -383,6 +385,7 @@ class AdminImpl : public Admin, private: const std::vector empty_access_logs_; std::unique_ptr init_manager_; + const bool ignore_global_conn_limit_; }; using AdminListenerPtr = std::unique_ptr; @@ -455,6 +458,7 @@ class AdminImpl : public Admin, const LocalReply::LocalReplyPtr local_reply_; const std::vector detection_extensions_{}; const absl::optional scheme_{}; + const bool ignore_global_conn_limit_; }; } // namespace Server diff --git a/source/server/config_validation/dispatcher.cc b/source/server/config_validation/dispatcher.cc index 155b482709e22..36d83abe8f1d7 100644 --- a/source/server/config_validation/dispatcher.cc +++ b/source/server/config_validation/dispatcher.cc @@ -16,7 +16,8 @@ Network::ClientConnectionPtr ValidationDispatcher::createClientConnection( } Network::ListenerPtr ValidationDispatcher::createListener(Network::SocketSharedPtr&&, - Network::TcpListenerCallbacks&, bool) { + Network::TcpListenerCallbacks&, bool, + bool) { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } diff --git a/source/server/config_validation/dispatcher.h b/source/server/config_validation/dispatcher.h index e16829cdb0b61..0a280d0b98a50 100644 --- a/source/server/config_validation/dispatcher.h +++ b/source/server/config_validation/dispatcher.h @@ -24,7 +24,7 @@ class ValidationDispatcher : public DispatcherImpl { Network::Address::InstanceConstSharedPtr, Network::TransportSocketPtr&&, const Network::ConnectionSocket::OptionsSharedPtr& options) override; Network::ListenerPtr createListener(Network::SocketSharedPtr&&, Network::TcpListenerCallbacks&, - bool bind_to_port) override; + bool bind_to_port, bool ignore_global_conn_limit) override; }; } // namespace Event diff --git a/source/server/configuration_impl.cc b/source/server/configuration_impl.cc index a4556846df540..d5abc1439b7ab 100644 --- a/source/server/configuration_impl.cc +++ b/source/server/configuration_impl.cc @@ -206,6 +206,7 @@ InitialImpl::InitialImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstra admin_.socket_options_, Network::SocketOptionFactory::buildLiteralOptions(admin.socket_options())); } + admin_.ignore_global_conn_limit_ = admin.ignore_global_conn_limit(); if (!bootstrap.flags_path().empty()) { flags_path_ = bootstrap.flags_path(); diff --git a/source/server/configuration_impl.h b/source/server/configuration_impl.h index 7efa5e22fda54..792a106e26f5d 100644 --- a/source/server/configuration_impl.h +++ b/source/server/configuration_impl.h @@ -192,11 +192,13 @@ class InitialImpl : public Initial { Network::Address::InstanceConstSharedPtr address() override { return address_; } Network::Socket::OptionsSharedPtr socketOptions() override { return socket_options_; } std::list accessLogs() const override { return access_logs_; } + bool ignoreGlobalConnLimit() const override { return ignore_global_conn_limit_; } std::string profile_path_; std::list access_logs_; Network::Address::InstanceConstSharedPtr address_; Network::Socket::OptionsSharedPtr socket_options_; + bool ignore_global_conn_limit_; }; AdminImpl admin_; diff --git a/source/server/listener_impl.cc b/source/server/listener_impl.cc index 1ed954ba5a9c3..e1d1788e9d161 100644 --- a/source/server/listener_impl.cc +++ b/source/server/listener_impl.cc @@ -309,6 +309,7 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, validation_visitor_( added_via_api_ ? parent_.server_.messageValidationContext().dynamicValidationVisitor() : parent_.server_.messageValidationContext().staticValidationVisitor()), + ignore_global_conn_limit_(config.ignore_global_conn_limit()), listener_init_target_(fmt::format("Listener-init-target {}", name), [this]() { dynamic_init_manager_->initialize(local_init_watcher_); }), dynamic_init_manager_(std::make_unique( @@ -399,6 +400,7 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, validation_visitor_( added_via_api_ ? parent_.server_.messageValidationContext().dynamicValidationVisitor() : parent_.server_.messageValidationContext().staticValidationVisitor()), + ignore_global_conn_limit_(config.ignore_global_conn_limit()), // listener_init_target_ is not used during in place update because we expect server started. listener_init_target_("", nullptr), dynamic_init_manager_(std::make_unique( diff --git a/source/server/listener_impl.h b/source/server/listener_impl.h index 14153bfaa7d6c..765631127c8e9 100644 --- a/source/server/listener_impl.h +++ b/source/server/listener_impl.h @@ -337,6 +337,7 @@ class ListenerImpl final : public Network::ListenerConfig, } uint32_t tcpBacklogSize() const override { return tcp_backlog_size_; } Init::Manager& initManager() override; + bool ignoreGlobalConnLimit() const override { return ignore_global_conn_limit_; } envoy::config::core::v3::TrafficDirection direction() const override { return config().traffic_direction(); } @@ -423,6 +424,7 @@ class ListenerImpl final : public Network::ListenerConfig, const uint64_t hash_; const uint32_t tcp_backlog_size_; ProtobufMessage::ValidationVisitor& validation_visitor_; + const bool ignore_global_conn_limit_; // A target is added to Server's InitManager if workers_started_ is false. Init::TargetImpl listener_init_target_; diff --git a/source/server/server.cc b/source/server/server.cc index 6879298a5a941..c914699197e6b 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -487,7 +487,8 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add // This is needed so that we don't read the value until runtime is fully initialized. enable_reuse_port_default_ = ReusePortDefault::Runtime; } - admin_ = std::make_unique(initial_config.admin().profilePath(), *this); + admin_ = std::make_unique(initial_config.admin().profilePath(), *this, + initial_config.admin().ignoreGlobalConnLimit()); loadServerFlags(initial_config.flagsPath()); diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 87fecd8ebe183..b3c873786e3a7 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -291,7 +291,8 @@ class CodecNetworkTest : public Event::TestUsingSimulatedTime, Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection( socket->connectionInfoProvider().localAddress(), source_address_, Network::Test::createRawBufferSocket(), nullptr); - upstream_listener_ = dispatcher_->createListener(std::move(socket), listener_callbacks_, true); + upstream_listener_ = + dispatcher_->createListener(std::move(socket), listener_callbacks_, true, false); client_connection_ = client_connection.get(); client_connection_->addConnectionCallbacks(client_callbacks_); diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index ed24bf50a0601..88b8f79e9e6b7 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -151,7 +151,7 @@ class ConnectionImplTest : public testing::TestWithParam { } socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); - listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true); + listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true, false); client_connection_ = std::make_unique( *dispatcher_, socket_->connectionInfoProvider().localAddress(), source_address_, Network::Test::createRawBufferSocket(), socket_options_); @@ -1370,7 +1370,7 @@ TEST_P(ConnectionImplTest, BindFailureTest) { dispatcher_ = api_->allocateDispatcher("test_thread"); socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); - listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true); + listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true, false); client_connection_ = dispatcher_->createClientConnection( socket_->connectionInfoProvider().localAddress(), source_address_, @@ -2841,7 +2841,7 @@ class ReadBufferLimitTest : public ConnectionImplTest { dispatcher_ = api_->allocateDispatcher("test_thread"); socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); - listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true); + listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true, false); client_connection_ = dispatcher_->createClientConnection(socket_->connectionInfoProvider().localAddress(), diff --git a/test/common/network/listener_impl_test.cc b/test/common/network/listener_impl_test.cc index b1f0aac6462d6..2e12e648be718 100644 --- a/test/common/network/listener_impl_test.cc +++ b/test/common/network/listener_impl_test.cc @@ -35,7 +35,8 @@ static void errorCallbackTest(Address::IpVersion version) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(version)); Network::MockTcpListenerCallbacks listener_callbacks; - Network::ListenerPtr listener = dispatcher->createListener(socket, listener_callbacks, true); + Network::ListenerPtr listener = + dispatcher->createListener(socket, listener_callbacks, true, false); Network::ClientConnectionPtr client_connection = dispatcher->createClientConnection( socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -66,8 +67,10 @@ TEST_P(ListenerImplDeathTest, ErrorCallback) { class TestTcpListenerImpl : public TcpListenerImpl { public: TestTcpListenerImpl(Event::DispatcherImpl& dispatcher, Random::RandomGenerator& random_generator, - SocketSharedPtr socket, TcpListenerCallbacks& cb, bool bind_to_port) - : TcpListenerImpl(dispatcher, random_generator, std::move(socket), cb, bind_to_port) {} + SocketSharedPtr socket, TcpListenerCallbacks& cb, bool bind_to_port, + bool ignore_global_conn_limit) + : TcpListenerImpl(dispatcher, random_generator, std::move(socket), cb, bind_to_port, + ignore_global_conn_limit) {} MOCK_METHOD(Address::InstanceConstSharedPtr, getLocalAddress, (os_fd_t fd)); }; @@ -85,10 +88,10 @@ TEST_P(TcpListenerImplTest, UseActualDst) { Random::MockRandomGenerator random_generator; // Do not redirect since use_original_dst is false. Network::TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, - listener_callbacks1, true); + listener_callbacks1, true, false); Network::MockTcpListenerCallbacks listener_callbacks2; Network::TestTcpListenerImpl listenerDst(dispatcherImpl(), random_generator, socketDst, - listener_callbacks2, false); + listener_callbacks2, false, false); Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection( socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -122,7 +125,8 @@ TEST_P(TcpListenerImplTest, GlobalConnectionLimitEnforcement) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(version_)); Network::MockTcpListenerCallbacks listener_callbacks; - Network::ListenerPtr listener = dispatcher_->createListener(socket, listener_callbacks, true); + Network::ListenerPtr listener = + dispatcher_->createListener(socket, listener_callbacks, true, false); std::vector client_connections; std::vector server_connections; @@ -181,6 +185,54 @@ TEST_P(TcpListenerImplTest, GlobalConnectionLimitEnforcement) { {{"overload.global_downstream_max_connections", ""}}); } +TEST_P(TcpListenerImplTest, GlobalConnectionLimitListenerOptOut) { + // Required to manipulate runtime values when there is no test server. + TestScopedRuntime scoped_runtime; + + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"overload.global_downstream_max_connections", "1"}}); + auto socket = std::make_shared( + Network::Test::getCanonicalLoopbackAddress(version_)); + Network::MockTcpListenerCallbacks listener_callbacks; + Network::ListenerPtr listener = + dispatcher_->createListener(socket, listener_callbacks, true, true); + + std::vector client_connections; + std::vector server_connections; + StreamInfo::StreamInfoImpl stream_info(dispatcher_->timeSource(), nullptr); + EXPECT_CALL(listener_callbacks, onAccept_(_)) + .WillRepeatedly(Invoke([&](Network::ConnectionSocketPtr& accepted_socket) -> void { + server_connections.emplace_back(dispatcher_->createServerConnection( + std::move(accepted_socket), Network::Test::createRawBufferSocket(), stream_info)); + dispatcher_->exit(); + })); + + auto initiate_connections = [&](const int count) { + for (int i = 0; i < count; ++i) { + client_connections.emplace_back( + dispatcher_->createClientConnection(socket->connectionInfoProvider().localAddress(), + Network::Address::InstanceConstSharedPtr(), + Network::Test::createRawBufferSocket(), nullptr)); + client_connections.back()->connect(); + } + }; + + initiate_connections(2); + EXPECT_CALL(listener_callbacks, onReject(TcpListenerCallbacks::RejectCause::GlobalCxLimit)) + .Times(0); + dispatcher_->run(Event::Dispatcher::RunType::Block); + + for (const auto& conn : client_connections) { + conn->close(ConnectionCloseType::NoFlush); + } + for (const auto& conn : server_connections) { + conn->close(ConnectionCloseType::NoFlush); + } + + // We expect any server-side connections that get created to populate 'server_connections'. + EXPECT_EQ(2, server_connections.size()); +} + TEST_P(TcpListenerImplTest, WildcardListenerUseActualDst) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(version_)); @@ -188,7 +240,7 @@ TEST_P(TcpListenerImplTest, WildcardListenerUseActualDst) { Random::MockRandomGenerator random_generator; // Do not redirect since use_original_dst is false. Network::TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, - listener_callbacks, true); + listener_callbacks, true, false); auto local_dst_address = Network::Utility::getAddressWithPort( *Network::Test::getCanonicalLoopbackAddress(version_), @@ -232,7 +284,7 @@ TEST_P(TcpListenerImplTest, WildcardListenerIpv4Compat) { // Do not redirect since use_original_dst is false. Network::TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, - listener_callbacks, true); + listener_callbacks, true, false); auto listener_address = Network::Utility::getAddressWithPort( *Network::Test::getCanonicalLoopbackAddress(version_), @@ -271,8 +323,8 @@ TEST_P(TcpListenerImplTest, DisableAndEnableListener) { MockTcpListenerCallbacks listener_callbacks; MockConnectionCallbacks connection_callbacks; Random::MockRandomGenerator random_generator; - TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, listener_callbacks, - true); + TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, listener_callbacks, true, + false); // When listener is disabled, the timer should fire before any connection is accepted. listener.disable(); @@ -312,8 +364,8 @@ TEST_P(TcpListenerImplTest, SetListenerRejectFractionZero) { MockTcpListenerCallbacks listener_callbacks; MockConnectionCallbacks connection_callbacks; Random::MockRandomGenerator random_generator; - TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, listener_callbacks, - true); + TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, listener_callbacks, true, + false); listener.setRejectFraction(UnitFloat(0)); @@ -343,8 +395,8 @@ TEST_P(TcpListenerImplTest, SetListenerRejectFractionIntermediate) { MockTcpListenerCallbacks listener_callbacks; MockConnectionCallbacks connection_callbacks; Random::MockRandomGenerator random_generator; - TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, listener_callbacks, - true); + TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, listener_callbacks, true, + false); listener.setRejectFraction(UnitFloat(0.5f)); @@ -406,8 +458,8 @@ TEST_P(TcpListenerImplTest, SetListenerRejectFractionAll) { MockTcpListenerCallbacks listener_callbacks; MockConnectionCallbacks connection_callbacks; Random::MockRandomGenerator random_generator; - TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, listener_callbacks, - true); + TestTcpListenerImpl listener(dispatcherImpl(), random_generator, socket, listener_callbacks, true, + false); listener.setRejectFraction(UnitFloat(1)); diff --git a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc index 705f80156db01..0d906a0b79287 100644 --- a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc +++ b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc @@ -85,6 +85,7 @@ class ProxyProtocolRegressionTest : public testing::TestWithParam { server_ = std::make_unique(*dispatcher_); socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); - listener_ = dispatcher_->createListener(socket_, *server_, true); + listener_ = dispatcher_->createListener(socket_, *server_, true, false); updateDnsResolverOptions(); // Create a resolver options on stack here to emulate what actually happens in envoy bootstrap. diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index af9288bbea316..283cf9109a304 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -342,7 +342,7 @@ void testUtil(const TestUtilOptions& options) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(options.version())); Network::MockTcpListenerCallbacks callbacks; - Network::ListenerPtr listener = dispatcher->createListener(socket, callbacks, true); + Network::ListenerPtr listener = dispatcher->createListener(socket, callbacks, true, false); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(options.clientCtxYaml()), @@ -677,7 +677,7 @@ void testUtilV2(const TestUtilOptionsV2& options) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(options.version())); NiceMock callbacks; - Network::ListenerPtr listener = dispatcher->createListener(socket, callbacks, true); + Network::ListenerPtr listener = dispatcher->createListener(socket, callbacks, true, false); Stats::TestUtil::TestStore client_stats_store; Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); @@ -2557,7 +2557,7 @@ TEST_P(SslSocketTest, FlushCloseDuringHandshake) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); Network::MockTcpListenerCallbacks callbacks; - Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true); + Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true, false); Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection( socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -2612,7 +2612,8 @@ TEST_P(SslSocketTest, HalfClose) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); Network::MockTcpListenerCallbacks listener_callbacks; - Network::ListenerPtr listener = dispatcher_->createListener(socket, listener_callbacks, true); + Network::ListenerPtr listener = + dispatcher_->createListener(socket, listener_callbacks, true, false); std::shared_ptr server_read_filter(new Network::MockReadFilter()); std::shared_ptr client_read_filter(new Network::MockReadFilter()); @@ -2693,7 +2694,8 @@ TEST_P(SslSocketTest, ShutdownWithCloseNotify) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); Network::MockTcpListenerCallbacks listener_callbacks; - Network::ListenerPtr listener = dispatcher_->createListener(socket, listener_callbacks, true); + Network::ListenerPtr listener = + dispatcher_->createListener(socket, listener_callbacks, true, false); std::shared_ptr server_read_filter(new Network::MockReadFilter()); std::shared_ptr client_read_filter(new Network::MockReadFilter()); @@ -2780,7 +2782,8 @@ TEST_P(SslSocketTest, ShutdownWithoutCloseNotify) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); Network::MockTcpListenerCallbacks listener_callbacks; - Network::ListenerPtr listener = dispatcher_->createListener(socket, listener_callbacks, true); + Network::ListenerPtr listener = + dispatcher_->createListener(socket, listener_callbacks, true, false); std::shared_ptr server_read_filter(new Network::MockReadFilter()); std::shared_ptr client_read_filter(new Network::MockReadFilter()); @@ -2883,7 +2886,7 @@ TEST_P(SslSocketTest, ClientAuthMultipleCAs) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); Network::MockTcpListenerCallbacks callbacks; - Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true); + Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true, false); const std::string client_ctx_yaml = R"EOF( common_tls_context: @@ -2980,8 +2983,8 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, Network::Test::getCanonicalLoopbackAddress(ip_version)); NiceMock callbacks; Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread")); - Network::ListenerPtr listener1 = dispatcher->createListener(socket1, callbacks, true); - Network::ListenerPtr listener2 = dispatcher->createListener(socket2, callbacks, true); + Network::ListenerPtr listener1 = dispatcher->createListener(socket1, callbacks, true, false); + Network::ListenerPtr listener2 = dispatcher->createListener(socket2, callbacks, true, false); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml), client_tls_context); @@ -3119,7 +3122,7 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml Network::Test::getCanonicalLoopbackAddress(ip_version)); NiceMock callbacks; Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread")); - Network::ListenerPtr listener = dispatcher->createListener(tcp_socket, callbacks, true); + Network::ListenerPtr listener = dispatcher->createListener(tcp_socket, callbacks, true, false); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml), client_tls_context); @@ -3561,8 +3564,8 @@ TEST_P(SslSocketTest, ClientAuthCrossListenerSessionResumption) { auto socket2 = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); Network::MockTcpListenerCallbacks callbacks; - Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true); - Network::ListenerPtr listener2 = dispatcher_->createListener(socket2, callbacks, true); + Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true, false); + Network::ListenerPtr listener2 = dispatcher_->createListener(socket2, callbacks, true, false); const std::string client_ctx_yaml = R"EOF( common_tls_context: tls_certificates: @@ -3679,7 +3682,7 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya NiceMock callbacks; Api::ApiPtr api = Api::createApiForTest(server_stats_store, time_system_); Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread")); - Network::ListenerPtr listener = dispatcher->createListener(socket, callbacks, true); + Network::ListenerPtr listener = dispatcher->createListener(socket, callbacks, true, false); Network::ConnectionPtr server_connection; Network::MockConnectionCallbacks server_connection_callbacks; @@ -3938,7 +3941,7 @@ TEST_P(SslSocketTest, SslError) { auto socket = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); Network::MockTcpListenerCallbacks callbacks; - Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true); + Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true, false); Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection( socket->connectionInfoProvider().localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -5023,7 +5026,7 @@ class SslReadBufferLimitTest : public SslSocketTest { socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam())); - listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true); + listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true, false); TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml_), upstream_tls_context_); auto client_cfg = diff --git a/test/integration/cx_limit_integration_test.cc b/test/integration/cx_limit_integration_test.cc index fd3c1bbb6fb1f..1be10dcd5c97f 100644 --- a/test/integration/cx_limit_integration_test.cc +++ b/test/integration/cx_limit_integration_test.cc @@ -31,8 +31,23 @@ class ConnectionLimitIntegrationTest : public testing::TestWithParammutable_listeners(0); + listener->set_ignore_global_conn_limit(true); + }); + } + + void setAdminGlobalLimitOptOut() { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* admin = bootstrap.mutable_admin(); + admin->set_ignore_global_conn_limit(true); + }); } void initialize() override { BaseIntegrationTest::initialize(); } @@ -95,9 +110,6 @@ class ConnectionLimitIntegrationTest : public testing::TestWithParamconnected()); const bool isV4 = (version_ == Network::Address::IpVersion::v4); - auto local_address = isV4 ? Network::Utility::getCanonicalIpv4LoopbackAddress() - : Network::Utility::getIpv6LoopbackAddress(); - const std::string counter_prefix = (isV4 ? "listener.127.0.0.1_0." : "listener.[__1]_0."); test_server_->waitForCounterEq(counter_prefix + check_stat, 1); @@ -145,7 +157,7 @@ TEST_P(ConnectionLimitIntegrationTest, TestGlobalLimit) { // Includes twice the number of connections expected because the tracking is performed via a // static variable and the fake upstream has a listener. This causes upstream connections to the // fake upstream to also be tracked as part of the global downstream connection tracking. - setGlobalLimit("4"); + setGlobalLimit(4); initialize(); }; @@ -154,9 +166,12 @@ TEST_P(ConnectionLimitIntegrationTest, TestGlobalLimit) { TEST_P(ConnectionLimitIntegrationTest, TestBothLimits) { std::function init_func = [this]() { + // Includes twice the number of connections expected because the tracking is performed via a + // static variable and the fake upstream has a listener. This causes upstream connections to the + // fake upstream to also be tracked as part of the global downstream connection tracking. + setGlobalLimit(4); // Setting the listener limit to a much higher value and making sure the right stat gets // incremented when both limits are set. - setGlobalLimit("4"); setListenerLimit(100); initialize(); }; @@ -164,5 +179,113 @@ TEST_P(ConnectionLimitIntegrationTest, TestBothLimits) { doTest(init_func, "downstream_global_cx_overflow"); } +TEST_P(ConnectionLimitIntegrationTest, TestGlobalLimitOptOut) { + // Includes 4 connections because the tracking is performed regardless of whether a specific + // listener has opted out. Since the fake upstream has a listener, we need to keep value at 4 so + // it can accept connections. (2 downstream listener conns + 2 upstream listener conns) + setGlobalLimit(4); + setGlobalLimitOptOut(); + setAdminGlobalLimitOptOut(); + setListenerLimit(100); + initialize(); + + std::vector tcp_clients; + std::vector raw_conns; + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + raw_conns.emplace_back(); + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(raw_conns.back())); + ASSERT_TRUE(tcp_clients.back()->connected()); + + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + raw_conns.emplace_back(); + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(raw_conns.back())); + ASSERT_TRUE(tcp_clients.back()->connected()); + + // 3rd connection should fail, not because listener_0 hit a limit, but because the + // upstream listener hit a limit (5 conns would exist when it goes to accept, so it rejects it). + // We can see that listener_0 didn't hit any limits because it's downstream_global_cx_overflow + // stat is still at 0, in contrast with the TestGlobalLimit test where it is 1 + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + raw_conns.emplace_back(); + ASSERT_FALSE( + fake_upstreams_[0]->waitForRawConnection(raw_conns.back(), std::chrono::milliseconds(500))); + tcp_clients.back()->waitForDisconnect(); + + // Get rid of the client that failed to connect. + tcp_clients.back()->close(); + tcp_clients.pop_back(); + + // admin connections should succeed + tcp_clients.emplace_back(makeTcpConnection(lookupPort("admin"))); + raw_conns.emplace_back(); + ASSERT_TRUE(tcp_clients.back()->connected()); + + const bool isV4 = (version_ == Network::Address::IpVersion::v4); + const std::string counter_prefix = (isV4 ? "listener.127.0.0.1_0." : "listener.[__1]_0."); + + // listener_0 does not hit any connection limits + test_server_->waitForCounterEq(counter_prefix + "downstream_global_cx_overflow", 0); + test_server_->waitForCounterEq(counter_prefix + "downstream_cx_overflow", 0); + test_server_->waitForCounterEq("listener.admin.downstream_global_cx_overflow", 0); + test_server_->waitForCounterEq("listener.admin.downstream_cx_overflow", 0); + + for (auto& tcp_client : tcp_clients) { + tcp_client->close(); + } + + tcp_clients.clear(); + raw_conns.clear(); +} + +TEST_P(ConnectionLimitIntegrationTest, TestListenerLimitWithGlobalOptOut) { + // Includes 4 connections because the tracking is performed regardless of whether a specific + // listener has opted out. Since the fake upstream has a listener, we need to keep value at 4 so + // it can accept connections. (2 downstream listener conns + 2 upstream listener conns) + setGlobalLimit(4); + setGlobalLimitOptOut(); + // Only allow 2 connections for the listener even though it has opted out of the global connection + // limit + setListenerLimit(2); + initialize(); + + std::vector tcp_clients; + std::vector raw_conns; + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + raw_conns.emplace_back(); + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(raw_conns.back())); + ASSERT_TRUE(tcp_clients.back()->connected()); + + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + raw_conns.emplace_back(); + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(raw_conns.back())); + ASSERT_TRUE(tcp_clients.back()->connected()); + + // 3rd connection should fail because we've hit the listener connection limit, not because + // we've hit a global limit + tcp_clients.emplace_back(makeTcpConnection(lookupPort("listener_0"))); + raw_conns.emplace_back(); + ASSERT_FALSE( + fake_upstreams_[0]->waitForRawConnection(raw_conns.back(), std::chrono::milliseconds(500))); + tcp_clients.back()->waitForDisconnect(); + + // Get rid of the client that failed to connect. + tcp_clients.back()->close(); + tcp_clients.pop_back(); + + const bool isV4 = (version_ == Network::Address::IpVersion::v4); + const std::string counter_prefix = (isV4 ? "listener.127.0.0.1_0." : "listener.[__1]_0."); + + // listener_0 does hits the listener connection limit + test_server_->waitForCounterEq(counter_prefix + "downstream_global_cx_overflow", 0); + test_server_->waitForCounterEq(counter_prefix + "downstream_cx_overflow", 1); + + for (auto& tcp_client : tcp_clients) { + tcp_client->close(); + } + + tcp_clients.clear(); + raw_conns.clear(); +} + } // namespace } // namespace Envoy diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 739bf54f88244..acb76221012a5 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -809,6 +809,7 @@ class FakeUpstream : Logger::Loggable, ResourceLimit& openConnections() override { return connection_resource_; } uint32_t tcpBacklogSize() const override { return ENVOY_TCP_BACKLOG_SIZE; } Init::Manager& initManager() override { return *init_manager_; } + bool ignoreGlobalConnLimit() const override { return false; } void setMaxConnections(const uint32_t num_connections) { connection_resource_.setMax(num_connections); diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index 8d91e35582916..a155a75e75d10 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -70,9 +70,10 @@ class MockDispatcher : public Dispatcher { } Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::TcpListenerCallbacks& cb, - bool bind_to_port) override { - return Network::ListenerPtr{createListener_(std::move(socket), cb, bind_to_port)}; + Network::TcpListenerCallbacks& cb, bool bind_to_port, + bool ignore_global_conn_limit) override { + return Network::ListenerPtr{ + createListener_(std::move(socket), cb, bind_to_port, ignore_global_conn_limit)}; } Network::UdpListenerPtr @@ -138,7 +139,7 @@ class MockDispatcher : public Dispatcher { MOCK_METHOD(Filesystem::Watcher*, createFilesystemWatcher_, ()); MOCK_METHOD(Network::Listener*, createListener_, (Network::SocketSharedPtr && socket, Network::TcpListenerCallbacks& cb, - bool bind_to_port)); + bool bind_to_port, bool ignore_global_conn_limit)); MOCK_METHOD(Network::UdpListener*, createUdpListener_, (Network::SocketSharedPtr socket, Network::UdpListenerCallbacks& cb, const envoy::config::core::v3::UdpSocketConfig& config)); diff --git a/test/mocks/event/wrapped_dispatcher.h b/test/mocks/event/wrapped_dispatcher.h index c36705cd457b3..634235cbbd632 100644 --- a/test/mocks/event/wrapped_dispatcher.h +++ b/test/mocks/event/wrapped_dispatcher.h @@ -60,9 +60,9 @@ class WrappedDispatcher : public Dispatcher { } Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::TcpListenerCallbacks& cb, - bool bind_to_port) override { - return impl_.createListener(std::move(socket), cb, bind_to_port); + Network::TcpListenerCallbacks& cb, bool bind_to_port, + bool ignore_global_conn_limit) override { + return impl_.createListener(std::move(socket), cb, bind_to_port, ignore_global_conn_limit); } Network::UdpListenerPtr diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index c48098fc06668..40e5c43edd2ad 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -434,6 +434,7 @@ class MockListenerConfig : public ListenerConfig { MOCK_METHOD(ResourceLimit&, openConnections, ()); MOCK_METHOD(uint32_t, tcpBacklogSize, (), (const)); MOCK_METHOD(Init::Manager&, initManager, ()); + MOCK_METHOD(bool, ignoreGlobalConnLimit, (), (const)); envoy::config::core::v3::TrafficDirection direction() const override { return envoy::config::core::v3::UNSPECIFIED; diff --git a/test/server/admin/admin_instance.cc b/test/server/admin/admin_instance.cc index d63b0887c7467..e05656d9b336d 100644 --- a/test/server/admin/admin_instance.cc +++ b/test/server/admin/admin_instance.cc @@ -9,7 +9,7 @@ namespace Server { AdminInstanceTest::AdminInstanceTest() : address_out_path_(TestEnvironment::temporaryPath("admin.address")), cpu_profile_path_(TestEnvironment::temporaryPath("envoy.prof")), - admin_(cpu_profile_path_, server_), request_headers_{{":path", "/"}}, + admin_(cpu_profile_path_, server_, false), request_headers_{{":path", "/"}}, admin_filter_(admin_.createCallbackFunction()) { std::list access_logs; Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, "/dev/null"}; diff --git a/test/server/admin/admin_test.cc b/test/server/admin/admin_test.cc index 8a0fce0464993..99b93afda1d4e 100644 --- a/test/server/admin/admin_test.cc +++ b/test/server/admin/admin_test.cc @@ -67,7 +67,7 @@ TEST_P(AdminInstanceTest, WriteAddressToFile) { TEST_P(AdminInstanceTest, AdminAddress) { std::string address_out_path = TestEnvironment::temporaryPath("admin.address"); - AdminImpl admin_address_out_path(cpu_profile_path_, server_); + AdminImpl admin_address_out_path(cpu_profile_path_, server_, false); std::list access_logs; Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, "/dev/null"}; access_logs.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( @@ -82,7 +82,7 @@ TEST_P(AdminInstanceTest, AdminAddress) { TEST_P(AdminInstanceTest, AdminBadAddressOutPath) { std::string bad_path = TestEnvironment::temporaryPath("some/unlikely/bad/path/admin.address"); - AdminImpl admin_bad_address_out_path(cpu_profile_path_, server_); + AdminImpl admin_bad_address_out_path(cpu_profile_path_, server_, false); std::list access_logs; Filesystem::FilePathAndType file_info{Filesystem::DestinationType::File, "/dev/null"}; access_logs.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( diff --git a/test/server/admin/profiling_handler_test.cc b/test/server/admin/profiling_handler_test.cc index 709675e00a11f..aefa0a6b10d7a 100644 --- a/test/server/admin/profiling_handler_test.cc +++ b/test/server/admin/profiling_handler_test.cc @@ -67,7 +67,7 @@ TEST_P(AdminInstanceTest, AdminHeapProfiler) { TEST_P(AdminInstanceTest, AdminBadProfiler) { Buffer::OwnedImpl data; AdminImpl admin_bad_profile_path(TestEnvironment::temporaryPath("some/unlikely/bad/path.prof"), - server_); + server_, false); Http::TestResponseHeaderMapImpl header_map; const absl::string_view post = Http::Headers::get().MethodValues.Post; request_headers_.setMethod(post); diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 74a5ca332db0b..d2b733072283b 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -70,7 +70,8 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable access_log, std::shared_ptr> filter_chain_manager = nullptr, uint32_t tcp_backlog_size = ENVOY_TCP_BACKLOG_SIZE, - Network::ConnectionBalancerSharedPtr connection_balancer = nullptr) + Network::ConnectionBalancerSharedPtr connection_balancer = nullptr, + bool ignore_global_conn_limit = false) : parent_(parent), socket_(std::make_shared>()), tag_(tag), bind_to_port_(bind_to_port), tcp_backlog_size_(tcp_backlog_size), hand_off_restored_destination_connections_(hand_off_restored_destination_connections), @@ -80,7 +81,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable() : connection_balancer), access_logs_({access_log}), inline_filter_chain_manager_(filter_chain_manager), - init_manager_(nullptr) { + init_manager_(nullptr), ignore_global_conn_limit_(ignore_global_conn_limit) { envoy::config::listener::v3::UdpListenerConfig udp_config; udp_listener_config_ = std::make_unique(udp_config); udp_listener_config_->listener_factory_ = @@ -149,6 +150,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable access_logs_; std::shared_ptr> inline_filter_chain_manager_; std::unique_ptr init_manager_; + const bool ignore_global_conn_limit_; envoy::config::core::v3::TrafficDirection direction_; Network::UdpListenerCallbacks* udp_listener_callbacks_{}; }; @@ -224,11 +227,12 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable> overridden_filter_chain_manager = nullptr, - uint32_t tcp_backlog_size = ENVOY_TCP_BACKLOG_SIZE) { + uint32_t tcp_backlog_size = ENVOY_TCP_BACKLOG_SIZE, bool ignore_global_conn_limit = false) { listeners_.emplace_back(std::make_unique( *this, tag, bind_to_port, hand_off_restored_destination_connections, name, socket_type, listener_filters_timeout, continue_on_listener_filters_timeout, access_log_, - overridden_filter_chain_manager, tcp_backlog_size, connection_balancer)); + overridden_filter_chain_manager, tcp_backlog_size, connection_balancer, + ignore_global_conn_limit)); if (listener == nullptr) { // Expecting listener config in place update. @@ -239,9 +243,9 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggablesocket_factory_, getListenSocket(_)) .WillOnce(Return(listeners_.back()->socket_)); if (socket_type == Network::Socket::Type::Stream) { - EXPECT_CALL(dispatcher_, createListener_(_, _, _)) + EXPECT_CALL(dispatcher_, createListener_(_, _, _, _)) .WillOnce(Invoke([listener, listener_callbacks](Network::SocketSharedPtr&&, - Network::TcpListenerCallbacks& cb, + Network::TcpListenerCallbacks& cb, bool, bool) -> Network::Listener* { if (listener_callbacks != nullptr) { *listener_callbacks = &cb; From a7a00d9e8d52e517c1db1fd98dc09b5d193c8585 Mon Sep 17 00:00:00 2001 From: danzh Date: Wed, 24 Nov 2021 13:14:30 -0500 Subject: [PATCH 14/51] quic: turn off GRO (#19088) Signed-off-by: Dan Zhang danzh@google.com Commit Message: hard code prefer_gro to false. The performance of GRO hasn't been evaluated yet, so it shouldn't be default on. Risk Level: low Testing: existing tests pass --- source/common/quic/envoy_quic_client_connection.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/quic/envoy_quic_client_connection.cc b/source/common/quic/envoy_quic_client_connection.cc index 31670c62606e3..8ad98aa0afb02 100644 --- a/source/common/quic/envoy_quic_client_connection.cc +++ b/source/common/quic/envoy_quic_client_connection.cc @@ -200,7 +200,7 @@ void EnvoyQuicClientConnection::onFileEvent(uint32_t events, if (connected() && (events & Event::FileReadyType::Read)) { Api::IoErrorPtr err = Network::Utility::readPacketsFromSocket( connection_socket.ioHandle(), *connection_socket.connectionInfoProvider().localAddress(), - *this, dispatcher_.timeSource(), true, packets_dropped_); + *this, dispatcher_.timeSource(), /*prefer_gro=*/false, packets_dropped_); if (err == nullptr) { // In the case where the path validation fails, the probing socket will be closed and its IO // events are no longer interesting. From 8134744ef21125f3788cec408511a59ec97745e6 Mon Sep 17 00:00:00 2001 From: ankatare Date: Thu, 25 Nov 2021 14:11:55 +0530 Subject: [PATCH 15/51] broken link path fix for items http_filters/grpc_json_transcoder_filter (#19101) Signed-off-by: Abhay Narayan Katare --- .../http/http_filters/_include/grpc-transcoder-filter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/configuration/http/http_filters/_include/grpc-transcoder-filter.yaml b/docs/root/configuration/http/http_filters/_include/grpc-transcoder-filter.yaml index 80a8c67e0f726..919d61296e0e5 100644 --- a/docs/root/configuration/http/http_filters/_include/grpc-transcoder-filter.yaml +++ b/docs/root/configuration/http/http_filters/_include/grpc-transcoder-filter.yaml @@ -21,7 +21,7 @@ static_resources: domains: ["*"] routes: # NOTE: by default, matching happens based on the gRPC route, and not on the incoming request path. - # Reference: https://www.envoyproxy.io/docs/envoy/latest/configuration/http_filters/grpc_json_transcoder_filter#route-configs-for-transcoded-requests + # Reference: https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/grpc_json_transcoder_filter#route-configs-for-transcoded-requests - match: {prefix: "/helloworld.Greeter"} route: {cluster: grpc, timeout: 60s} http_filters: From a042c136db62c2cc00fa166abd7b66d8c848630d Mon Sep 17 00:00:00 2001 From: xuhj Date: Fri, 26 Nov 2021 16:52:52 +0800 Subject: [PATCH 16/51] Remove requested_server_name_ field from StreamInfo (#19102) Since the requested_server_name_ was moved to ConnectionInfoProvider, this field is useless now. Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: n/a Platform Specific Features: no Signed-off-by: He Jie Xu --- source/common/stream_info/stream_info_impl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/stream_info/stream_info_impl.h b/source/common/stream_info/stream_info_impl.h index bbf1bef50e4b8..0077ace81f281 100644 --- a/source/common/stream_info/stream_info_impl.h +++ b/source/common/stream_info/stream_info_impl.h @@ -468,7 +468,6 @@ struct StreamInfoImpl : public StreamInfo { uint64_t bytes_sent_{}; Network::Address::InstanceConstSharedPtr legacy_upstream_local_address_; const Network::ConnectionInfoProviderSharedPtr downstream_connection_info_provider_; - std::string requested_server_name_; const Http::RequestHeaderMap* request_headers_{}; Http::RequestIdStreamInfoProviderSharedPtr request_id_provider_; absl::optional downstream_timing_; From bfde85ee8d0209c982bf5f1e003b3e0ac45936d7 Mon Sep 17 00:00:00 2001 From: Faseela K Date: Fri, 26 Nov 2021 10:02:25 +0100 Subject: [PATCH 17/51] dep: Remove dependency - six (#19085) An internal 3pp scan reports that six version is older and a newer version is available. However as six is no longer used, attempting to remove it. Risk Level: low Testing: local build and pre checks done Signed-off-by: Faseela K --- bazel/repositories.bzl | 4 ---- bazel/repository_locations.bzl | 10 ---------- 2 files changed, 14 deletions(-) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 22dbc2c6b92e9..e2173ae264a85 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -115,10 +115,6 @@ def _python_deps(): name = "com_github_twitter_common_finagle_thrift", build_file = "@envoy//bazel/external:twitter_common_finagle_thrift.BUILD", ) - external_http_archive( - name = "six", - build_file = "@com_google_protobuf//third_party:six.BUILD", - ) # Bazel native C++ dependencies. For the dependencies that doesn't provide autoconf/automake builds. def _cc_deps(): diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 1279bc5d24e64..0abcdf18d2381 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -705,16 +705,6 @@ REPOSITORY_LOCATIONS_SPEC = dict( use_category = ["build"], release_date = "2021-10-22", ), - six = dict( - project_name = "Six", - project_desc = "Python 2 and 3 compatibility library", - project_url = "https://github.com/benjaminp/six", - version = "1.12.0", - sha256 = "0ce7aef70d066b8dda6425c670d00c25579c3daad8108b3e3d41bef26003c852", - urls = ["https://github.com/benjaminp/six/archive/{version}.tar.gz"], - release_date = "2018-12-10", - use_category = ["other"], - ), org_llvm_llvm = dict( # When changing this, you must re-generate the list of llvm libs # see `bazel/foreign_cc/BUILD` for further information. From 671b783f94318482141bb4de296134763f467bbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Nov 2021 14:43:09 +0000 Subject: [PATCH 18/51] build(deps): bump slack-sdk in /.github/actions/pr_notifier (#19093) Bumps [slack-sdk](https://github.com/slackapi/python-slack-sdk) from 3.11.2 to 3.12.0. - [Release notes](https://github.com/slackapi/python-slack-sdk/releases) - [Commits](https://github.com/slackapi/python-slack-sdk/compare/v3.11.2...v3.12.0) --- updated-dependencies: - dependency-name: slack-sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/actions/pr_notifier/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/pr_notifier/requirements.txt b/.github/actions/pr_notifier/requirements.txt index fb21f429db9fa..ed0d19437a68e 100644 --- a/.github/actions/pr_notifier/requirements.txt +++ b/.github/actions/pr_notifier/requirements.txt @@ -111,9 +111,9 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via pynacl -slack_sdk==3.11.2 \ - --hash=sha256:131bf605894525c2d66da064677eabc19f53f02ce0f82a3f2fa130d4ec3bc1b0 \ - --hash=sha256:35245ec34c8549fbb5c43ccc17101afd725b3508bb784da46530b214f496bf93 +slack_sdk==3.12.0 \ + --hash=sha256:a384d91c10229f94a9b2cae2ec5af2a683a3d5aee1287c01238630ab42747287 \ + --hash=sha256:f779ff3dc266491b02ad056d28038ec5d708b2a438a3a8f8794fb1121d8274e2 # via -r requirements.in urllib3==1.26.6 \ --hash=sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4 \ From 1260a04e1e34d6f8faf9515dc3adaa5949baad12 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Nov 2021 14:43:56 +0000 Subject: [PATCH 19/51] build(deps): bump charset-normalizer in /tools/dependency (#19105) Bumps [charset-normalizer](https://github.com/ousret/charset_normalizer) from 2.0.7 to 2.0.8. - [Release notes](https://github.com/ousret/charset_normalizer/releases) - [Changelog](https://github.com/Ousret/charset_normalizer/blob/master/CHANGELOG.md) - [Commits](https://github.com/ousret/charset_normalizer/compare/2.0.7...2.0.8) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/dependency/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/dependency/requirements.txt b/tools/dependency/requirements.txt index 9eb48be5c543d..1cec4a13c7e8a 100644 --- a/tools/dependency/requirements.txt +++ b/tools/dependency/requirements.txt @@ -60,9 +60,9 @@ cffi==1.15.0 \ --hash=sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997 \ --hash=sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796 # via pynacl -charset-normalizer==2.0.7 \ - --hash=sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0 \ - --hash=sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b +charset-normalizer==2.0.8 \ + --hash=sha256:83fcdeb225499d6344c8f7f34684c2981270beacc32ede2e669e94f7fa544405 \ + --hash=sha256:735e240d9a8506778cd7a453d97e817e536bb1fc29f4f6961ce297b9c7a917b0 # via requests colorama==0.4.4 \ --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \ From 29569d89499643d070a8b342cf64c13a807c2b59 Mon Sep 17 00:00:00 2001 From: Adam Kotwasinski Date: Fri, 26 Nov 2021 07:17:47 -0800 Subject: [PATCH 20/51] kafka: dependency upgrades (#18995) * kafka: upgrade librdkafka (used by mesh-filter) * kafka: upgrade dpkp (used in broker- & mesh-filter integration tests) * kafka: upgrade kafka server binary (user in broker- & mesh-filter integration tests) * kafka: upgrade kafka dependency (used to generated protocol code) Signed-off-by: Adam Kotwasinski --- bazel/repository_locations.bzl | 22 +++++++++---------- .../network/source/protocol/generator.py | 5 ----- .../network_filters/kafka_broker_filter.rst | 4 ++-- .../network_filters/kafka_mesh_filter.rst | 2 +- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 0abcdf18d2381..55ca4c04039c4 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -939,45 +939,45 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Kafka (source)", project_desc = "Open-source distributed event streaming platform", project_url = "https://kafka.apache.org", - version = "2.8.1", - sha256 = "c3fd89257e056e11b5e1b09d4bbd8332ce5abfdfa7c7a5bb6a5cfe9860fcc688", + version = "3.0.0", + sha256 = "862526ee07c372d7b2f7e672c096fe84bb1e115ef536e0ad7497e6fb50e08e02", strip_prefix = "kafka-{version}/clients/src/main/resources/common/message", urls = ["https://github.com/apache/kafka/archive/{version}.zip"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.network.kafka_broker", "envoy.filters.network.kafka_mesh"], - release_date = "2021-09-14", + release_date = "2021-09-08", cpe = "cpe:2.3:a:apache:kafka:*", ), edenhill_librdkafka = dict( project_name = "Kafka (C/C++ client)", project_desc = "C/C++ client for Apache Kafka (open-source distributed event streaming platform)", project_url = "https://github.com/edenhill/librdkafka", - version = "1.7.0", - sha256 = "c71b8c5ff419da80c31bb8d3036a408c87ad523e0c7588e7660ee5f3c8973057", + version = "1.8.2", + sha256 = "6a747d293a7a4613bd2897e28e8791476fbe1ae7361f2530a876e0fd483482a6", strip_prefix = "librdkafka-{version}", urls = ["https://github.com/edenhill/librdkafka/archive/v{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.filters.network.kafka_mesh"], - release_date = "2021-05-10", + release_date = "2021-10-18", cpe = "N/A", ), kafka_server_binary = dict( project_name = "Kafka (server binary)", project_desc = "Open-source distributed event streaming platform", project_url = "https://kafka.apache.org", - version = "2.8.1", - sha256 = "4888b03e3b27dd94f2d830ce3bae9d7d98b0ccee3a5d30c919ccb60e0fa1f139", + version = "3.0.0", + sha256 = "a82728166bbccf406009747a25e1fe52dbcb4d575e4a7a8616429b5818cd02d1", strip_prefix = "kafka_2.13-{version}", urls = ["https://archive.apache.org/dist/kafka/{version}/kafka_2.13-{version}.tgz"], - release_date = "2021-09-14", + release_date = "2021-09-20", use_category = ["test_only"], ), kafka_python_client = dict( project_name = "Kafka (Python client)", project_desc = "Open-source distributed event streaming platform", project_url = "https://kafka.apache.org", - version = "2.0.1", - sha256 = "05f7c6eecb402f11fcb7e524c903f1ba1c38d3bdc9bf42bc8ec3cf7567b9f979", + version = "2.0.2", + sha256 = "5dcf87c559e7aee4f18d621a02e247db3e3552ee4589ca611d51eef87b37efed", strip_prefix = "kafka-python-{version}", urls = ["https://github.com/dpkp/kafka-python/archive/{version}.tar.gz"], release_date = "2020-09-30", diff --git a/contrib/kafka/filters/network/source/protocol/generator.py b/contrib/kafka/filters/network/source/protocol/generator.py index 2fd18ebc2d69b..05ab8d0599669 100755 --- a/contrib/kafka/filters/network/source/protocol/generator.py +++ b/contrib/kafka/filters/network/source/protocol/generator.py @@ -126,12 +126,7 @@ def parse_messages(self, input_files): r'^\s*$', '', without_comments, flags=re.MULTILINE) # Windows support: see PR 10542 for details. amended = re.sub(r'-2147483648', 'INT32_MIN', without_empty_newlines) - # Kafka JSON files are malformed. See KAFKA-12794. - if input_file == 'external/kafka_source/DescribeProducersRequest.json': - amended = amended[:-6] message_spec = json.loads(amended) - # Adopt publicly available messages only: - # https://kafka.apache.org/28/protocol.html#protocol_api_keys api_key = message_spec['apiKey'] if api_key <= 51 or api_key in [56, 57, 60, 61]: message = self.parse_top_level_element(message_spec) diff --git a/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst b/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst index ec7828db5c120..8608fd3669162 100644 --- a/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst +++ b/docs/root/configuration/listeners/network_filters/kafka_broker_filter.rst @@ -5,8 +5,8 @@ Kafka Broker filter The Apache Kafka broker filter decodes the client protocol for `Apache Kafka `_, both the requests and responses in the payload. -The message versions in `Kafka 2.8.1 `_ -are supported. +The message versions in `Kafka 3.0.0 `_ +are supported (apart from API keys 65-67 which were introduced recently). The filter attempts not to influence the communication between client and brokers, so the messages that could not be decoded (due to Kafka client or broker running a newer version than supported by this filter) are forwarded as-is. diff --git a/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst b/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst index bebb7c31aa5bf..2ad5bd2c2924e 100644 --- a/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst +++ b/docs/root/configuration/listeners/network_filters/kafka_mesh_filter.rst @@ -6,7 +6,7 @@ Kafka Mesh filter The Apache Kafka mesh filter provides a facade for `Apache Kafka `_ producers. Produce requests sent to this filter insance can be forwarded to one of multiple clusters, depending on configured forwarding rules. Corresponding message versions from -Kafka 2.8.1 are supported. +Kafka 3.0.0 are supported. * :ref:`v3 API reference ` * This filter should be configured with the name *envoy.filters.network.kafka_mesh*. From 2950cf0afd4bfe48a72d8c475262305c0e258ba1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Nov 2021 23:02:51 -0500 Subject: [PATCH 21/51] build(deps): bump frozendict from 2.0.7 to 2.1.0 in /tools/base (#19080) Bumps [frozendict](https://github.com/Marco-Sulla/python-frozendict) from 2.0.7 to 2.1.0. - [Release notes](https://github.com/Marco-Sulla/python-frozendict/releases) - [Commits](https://github.com/Marco-Sulla/python-frozendict/compare/v2.0.7...v2.1.0) --- updated-dependencies: - dependency-name: frozendict dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 33de28812db80..14a4394e51a04 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -332,9 +332,9 @@ flake8-polyfill==1.0.2 \ --hash=sha256:12be6a34ee3ab795b19ca73505e7b55826d5f6ad7230d31b18e106400169b9e9 \ --hash=sha256:e44b087597f6da52ec6393a709e7108b2905317d0c0b744cdca6208e670d8eda # via pep8-naming -frozendict==2.0.7 \ - --hash=sha256:a68f609d1af67da80b45519fdcfca2d60249c0a8c96e68279c1b6ddd92128204 \ - --hash=sha256:d650f9cf3d2c5438b1631488a975a49b3bdd12c7a97eec59b85e57821eebf28a +frozendict==2.1.0 \ + --hash=sha256:0189168749ddea8601afd648146c502533f93ae33840eb76cd71f694742623cd \ + --hash=sha256:1c0de90a47fcfb9abf18458cb82573e817455e45f9e6a250f8988e6725f3973a # via # -r requirements.in # envoy.base.runner From 56a32584d11cf8a34ee9679d747e14a16a16ebb8 Mon Sep 17 00:00:00 2001 From: Takeshi Yoneda Date: Mon, 29 Nov 2021 13:04:41 +0900 Subject: [PATCH 22/51] dep: update Proxy-Wasm C++ host (2021-11-18). (#19074) Signed-off-by: mathetake --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 55ca4c04039c4..16207ad99e713 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1011,8 +1011,8 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "WebAssembly for Proxies (C++ host implementation)", project_desc = "WebAssembly for Proxies (C++ host implementation)", project_url = "https://github.com/proxy-wasm/proxy-wasm-cpp-host", - version = "e0636f7119f5fb15841799289a7e3558d7234374", - sha256 = "c5ea68cba0fda6d81eb007dccaf0433839c6e5b6185c9e1a40945a1b67ab1382", + version = "f38347360feaaf5b2a733f219c4d8c9660d626f0", + sha256 = "bf10de946eb5785813895c2bf16504afc0cd590b9655d9ee52fb1074d0825ea3", strip_prefix = "proxy-wasm-cpp-host-{version}", urls = ["https://github.com/proxy-wasm/proxy-wasm-cpp-host/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], @@ -1028,7 +1028,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( "envoy.wasm.runtime.wavm", "envoy.wasm.runtime.wasmtime", ], - release_date = "2021-11-12", + release_date = "2021-11-18", cpe = "N/A", ), proxy_wasm_rust_sdk = dict( From 2bf847854610db8bc5a44ef3046fcc8f3a23518e Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 29 Nov 2021 08:29:02 -0500 Subject: [PATCH 23/51] tcp: fix overenthusiastic bounds on the new pool (#19036) Risk Level: Low Testing: new integration test Docs Changes: made API more clear when requests count as connections :-/ Release Notes: inline Fixes #19033 Signed-off-by: Alyssa Wilk --- .../config/cluster/v3/circuit_breaker.proto | 2 ++ docs/root/version_history/current.rst | 1 + source/common/conn_pool/conn_pool_base.cc | 2 +- source/common/conn_pool/conn_pool_base.h | 4 ++++ source/common/tcp/conn_pool.h | 1 + test/integration/tcp_proxy_integration_test.cc | 16 ++++++++++++++++ 6 files changed, 25 insertions(+), 1 deletion(-) diff --git a/api/envoy/config/cluster/v3/circuit_breaker.proto b/api/envoy/config/cluster/v3/circuit_breaker.proto index 82cd329b91a72..34c907a5013ac 100644 --- a/api/envoy/config/cluster/v3/circuit_breaker.proto +++ b/api/envoy/config/cluster/v3/circuit_breaker.proto @@ -59,10 +59,12 @@ message CircuitBreakers { // The maximum number of pending requests that Envoy will allow to the // upstream cluster. If not specified, the default is 1024. + // This limit is applied as a connection limit for non-HTTP traffic. google.protobuf.UInt32Value max_pending_requests = 3; // The maximum number of parallel requests that Envoy will make to the // upstream cluster. If not specified, the default is 1024. + // This limit does not apply to non-HTTP traffic. google.protobuf.UInt32Value max_requests = 4; // The maximum number of parallel retries that Envoy will allow to the diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 93acb4131cdd7..0830081fae39c 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -27,6 +27,7 @@ Bug Fixes * ext_authz: fix the ext_authz network filter to correctly set response flag and code details to ``UAEX`` when a connection is denied. * listener: fixed the crash when updating listeners that do not bind to port. +* tcp: fixed a bug where upstream circuit breakers applied HTTP per-request bounds to TCP connections. * thrift_proxy: fix the thrift_proxy connection manager to correctly report success/error response metrics when performing :ref:`payload passthrough `. Removed Config or Runtime diff --git a/source/common/conn_pool/conn_pool_base.cc b/source/common/conn_pool/conn_pool_base.cc index 6a3f603148003..11b65b5555866 100644 --- a/source/common/conn_pool/conn_pool_base.cc +++ b/source/common/conn_pool/conn_pool_base.cc @@ -166,7 +166,7 @@ void ConnPoolImplBase::attachStreamToClient(Envoy::ConnectionPool::ActiveClient& AttachContext& context) { ASSERT(client.state() == Envoy::ConnectionPool::ActiveClient::State::READY); - if (!host_->cluster().resourceManager(priority_).requests().canCreate()) { + if (enforceMaxRequests() && !host_->cluster().resourceManager(priority_).requests().canCreate()) { ENVOY_LOG(debug, "max streams overflow"); onPoolFailure(client.real_host_description_, absl::string_view(), ConnectionPool::PoolFailureReason::Overflow, context); diff --git a/source/common/conn_pool/conn_pool_base.h b/source/common/conn_pool/conn_pool_base.h index 802cff1d639c2..4bdf01a5aec01 100644 --- a/source/common/conn_pool/conn_pool_base.h +++ b/source/common/conn_pool/conn_pool_base.h @@ -323,6 +323,10 @@ class ConnPoolImplBase : protected Logger::Loggable { const Network::ConnectionSocket::OptionsSharedPtr socket_options_; const Network::TransportSocketOptionsConstSharedPtr transport_socket_options_; + // True if the max requests circuit breakers apply. + // This will be false for the TCP pool, true otherwise. + virtual bool enforceMaxRequests() const { return true; } + std::list idle_callbacks_; // When calling purgePendingStreams, this list will be used to hold the streams we are about diff --git a/source/common/tcp/conn_pool.h b/source/common/tcp/conn_pool.h index d3a4a6d81d37e..760ed61e1dcef 100644 --- a/source/common/tcp/conn_pool.h +++ b/source/common/tcp/conn_pool.h @@ -217,6 +217,7 @@ class ConnPoolImpl : public Envoy::ConnectionPool::ConnPoolImplBase, callbacks->onPoolFailure(reason, failure_reason, host_description); } + bool enforceMaxRequests() const override { return false; } // These two functions exist for testing parity between old and new Tcp Connection Pools. virtual void onConnReleased(Envoy::ConnectionPool::ActiveClient&) {} virtual void onConnDestroyed() {} diff --git a/test/integration/tcp_proxy_integration_test.cc b/test/integration/tcp_proxy_integration_test.cc index 691bcd8df6118..435a7884b2191 100644 --- a/test/integration/tcp_proxy_integration_test.cc +++ b/test/integration/tcp_proxy_integration_test.cc @@ -172,8 +172,24 @@ TEST_P(TcpProxyIntegrationTest, TcpProxyDownstreamDisconnect) { TEST_P(TcpProxyIntegrationTest, TcpProxyManyConnections) { autonomous_upstream_ = true; + config_helper_.addConfigModifier([&](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* static_resources = bootstrap.mutable_static_resources(); + for (int i = 0; i < static_resources->clusters_size(); ++i) { + auto* cluster = static_resources->mutable_clusters(i); + auto* thresholds = cluster->mutable_circuit_breakers()->add_thresholds(); + thresholds->mutable_max_connections()->set_value(1027); + thresholds->mutable_max_pending_requests()->set_value(1027); + } + }); initialize(); +// The large number of connection is meant to regression test +// https://github.com/envoyproxy/envoy/issues/19033 but fails on apple CI +// TODO(alyssawilk) debug. +#if defined(__APPLE__) const int num_connections = 50; +#else + const int num_connections = 1026; +#endif std::vector clients(num_connections); for (int i = 0; i < num_connections; ++i) { From e3cb8ee8c91dd29243290a1c346b12f6a5f65d1c Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 29 Nov 2021 14:49:18 +0000 Subject: [PATCH 24/51] cve_scan: Use `envoy.dependency.cve_scan` (#19047) Signed-off-by: Ryan Northey --- ci/do_ci.sh | 6 - tools/base/requirements.in | 1 + tools/base/requirements.txt | 13 ++ tools/dependency/BUILD | 21 +- tools/dependency/cve.yaml | 34 +--- tools/dependency/cve_scan.py | 307 ++++++------------------------ tools/dependency/cve_scan_test.py | 295 ---------------------------- 7 files changed, 80 insertions(+), 597 deletions(-) delete mode 100755 tools/dependency/cve_scan_test.py diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 4334c2304b34c..d7a0c90e79348 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -462,7 +462,6 @@ elif [[ "$CI_TARGET" == "deps" ]]; then exit 0 elif [[ "$CI_TARGET" == "cve_scan" ]]; then echo "scanning for CVEs in dependencies..." - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:cve_scan_test bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:cve_scan exit 0 elif [[ "$CI_TARGET" == "tooling" ]]; then @@ -482,11 +481,6 @@ elif [[ "$CI_TARGET" == "tooling" ]]; then echo "dependency validate_test..." "${ENVOY_SRCDIR}"/tools/dependency/validate_test.py - # Validate the CVE scanner works. We do it here as well as in cve_scan, since this blocks - # presubmits, but cve_scan only runs async. - echo "cve_scan_test..." - bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/dependency:cve_scan_test - exit 0 elif [[ "$CI_TARGET" == "verify_examples" ]]; then run_ci_verify "*" "wasm-cc|win32-front-proxy" diff --git a/tools/base/requirements.in b/tools/base/requirements.in index 8261cdecf553a..8041aa7fb3d78 100644 --- a/tools/base/requirements.in +++ b/tools/base/requirements.in @@ -8,6 +8,7 @@ envoy.base.checker envoy.base.runner envoy.base.utils>=0.0.10 envoy.code_format.python_check>=0.0.4 +envoy.dependency.cve_scan envoy.dependency.pip_check>=0.0.4 envoy.distribution.release envoy.distribution.verify diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 14a4394e51a04..1cb42f984abb5 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -11,6 +11,7 @@ abstracts==0.0.12 \ # envoy.abstract.command # envoy.base.utils # envoy.code-format.python-check + # envoy.dependency.cve-scan # envoy.dependency.pip-check # envoy.github.abstract # envoy.github.release @@ -19,6 +20,7 @@ aio.functional==0.0.9 \ # via # -r requirements.in # aio.tasks + # envoy.dependency.cve-scan # envoy.github.abstract # envoy.github.release aio.stream==0.0.2 \ @@ -34,6 +36,7 @@ aio.tasks==0.0.4 \ # via # -r requirements.in # envoy.code-format.python-check + # envoy.dependency.cve-scan # envoy.github.abstract # envoy.github.release aiodocker==0.21.0 \ @@ -87,6 +90,7 @@ aiohttp==3.7.4.post0 \ # via # aio.stream # aiodocker + # envoy.dependency.cve-scan # envoy.github.abstract # envoy.github.release # slackclient @@ -261,6 +265,7 @@ envoy.base.checker==0.0.2 \ # via # -r requirements.in # envoy.code-format.python-check + # envoy.dependency.cve-scan # envoy.dependency.pip-check # envoy.distribution.distrotest # envoy.distribution.verify @@ -279,6 +284,7 @@ envoy.base.utils==0.0.10 \ # via # -r requirements.in # envoy.code-format.python-check + # envoy.dependency.cve-scan # envoy.dependency.pip-check # envoy.distribution.distrotest # envoy.docs.sphinx-runner @@ -287,6 +293,10 @@ envoy.base.utils==0.0.10 \ envoy.code-format.python-check==0.0.4 \ --hash=sha256:5e166102d1f873f0c14640bcef87b46147cbad1cb68888c977acfde7fce96e04 # via -r requirements.in +envoy.dependency.cve-scan==0.0.1 \ + --hash=sha256:438973e6258deb271d60a9ad688c13ebf9c5360ccb9b6b0d4af3b3228235b153 \ + --hash=sha256:733fa5c6bdbe91da4afe1d46bca75279f717e410693866825d92208fa0d3418f + # via -r requirements.in envoy.dependency.pip-check==0.0.4 \ --hash=sha256:3213d77959f65c3c97e9b5d74cb14c02bc02dae64bac2e7c3cb829a2f4e5e40e # via -r requirements.in @@ -375,6 +385,7 @@ jinja2==3.0.3 \ --hash=sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7 # via # -r requirements.in + # envoy.dependency.cve-scan # sphinx markupsafe==2.0.1 \ --hash=sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298 \ @@ -481,6 +492,7 @@ packaging==21.0 \ --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \ --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 # via + # envoy.dependency.cve-scan # envoy.github.release # pytest # sphinx @@ -732,6 +744,7 @@ typing-extensions==3.10.0.2 \ # via # aiodocker # aiohttp + # gitpython uritemplate==3.0.1 \ --hash=sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f \ --hash=sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae diff --git a/tools/dependency/BUILD b/tools/dependency/BUILD index c2cfa4e8e17e4..394ad20c0a340 100644 --- a/tools/dependency/BUILD +++ b/tools/dependency/BUILD @@ -24,23 +24,12 @@ py_library( py_binary( name = "cve_scan", - srcs = [ - "cve_scan.py", - "utils.py", - ], + srcs = ["cve_scan.py"], args = ["$(location :cve.yaml)"], - data = [ - ":cve.yaml", - ":exports", - requirement("envoy.base.utils"), - ], -) - -py_binary( - name = "cve_scan_test", - srcs = ["cve_scan_test.py"], - data = [ - ":cve_scan", + data = [":cve.yaml"], + deps = [ + ":utils", + requirement("envoy.dependency.cve_scan"), ], ) diff --git a/tools/dependency/cve.yaml b/tools/dependency/cve.yaml index 25640f1decd94..45fc35106e3ea 100644 --- a/tools/dependency/cve.yaml +++ b/tools/dependency/cve.yaml @@ -2,11 +2,9 @@ # We only look back a few years, since we shouldn't have any ancient deps. start_year: 2018 -ndist_url: https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{year}.json.gz - # These CVEs are false positives for the match heuristics. An explanation is # required when adding a new entry to this list as a comment. -ignore: +ignored_cves: # Node.js issue unrelated to http-parser (napi_ API implementation). - CVE-2020-8174 # Node.js HTTP desync attack. Request smuggling due to CR and hyphen @@ -22,24 +20,12 @@ ignore: - CVE-2020-8251 # Node.js issue unrelated to http-parser (libuv). - CVE-2020-8252 -# Fixed via the nghttp2 1.41.0 bump in Envoy 8b6ea4. -- CVE-2020-11080 # Node.js issue rooted in a c-ares bug. Does not appear to affect # http-parser or our use of c-ares, c-ares has been bumped regardless. - CVE-2020-8277 -# gRPC issue that only affects Javascript bindings. -- CVE-2020-7768 # Node.js issue unrelated to http-parser, see # https://github.com/mhart/StringStream/issues/7. - CVE-2018-21270 -# These should not affect Curl 7.74.0, but we see false positives due to the -# relative release date and CPE wildcard. -- CVE-2020-8169 -- CVE-2020-8177 -- CVE-2020-8284 -# Low severity Curl issue with incorrect re-use of connections due to case -# in/sensitivity -- CVE-2021-22924 # Node.js issue unrelated to http-parser (Node TLS). - CVE-2020-8265 # Node.js request smuggling. @@ -51,9 +37,6 @@ ignore: # Node.js issue unrelated to http-parser (*). - CVE-2021-22883 - CVE-2021-22884 -# False positive on the match heuristic, fixed in Curl 7.76.0. -- CVE-2021-22876 -- CVE-2021-22890 # Node.js issues unrelated to http-parser. # See https://nvd.nist.gov/vuln/detail/CVE-2021-22918 # See https://nvd.nist.gov/vuln/detail/CVE-2021-22921 @@ -66,18 +49,3 @@ ignore: - CVE-2021-22931 - CVE-2021-22939 - CVE-2021-22940 -# -# Currently, cvescan does not respect/understand versions (see #18354). -# -# The following CVEs target versions that are not currently used in the Envoy repo. -# -# libcurl -- CVE-2021-22945 -# -# kafka -- CVE-2021-38153 -# -# wasmtime -- CVE-2021-39216 -- CVE-2021-39218 -- CVE-2021-39219 diff --git a/tools/dependency/cve_scan.py b/tools/dependency/cve_scan.py index 315780ae33010..fdd0d42b4c8bd 100755 --- a/tools/dependency/cve_scan.py +++ b/tools/dependency/cve_scan.py @@ -1,275 +1,88 @@ #!/usr/bin/env python3 -# Scan for any external dependencies that were last updated before known CVEs -# (and near relatives). We also try a fuzzy match on version information. +# usage +# +# with bazel: +# +# $ bazel run //tools/dependency:cve_scan -- -h +# +# $ bazel run //tools/dependency:cve_scan +# +# +# The upstream lib is maintained here: +# +# https://github.com/envoyproxy/pytooling/tree/main/envoy.dependency.cve_scan +# +# Please submit issues/PRs to the pytooling repo: +# +# https://github.com/envoyproxy/pytooling +# -from collections import defaultdict, namedtuple -import datetime as dt -import gzip -import json -import re import sys -import textwrap -import urllib.request +from functools import cached_property +from typing import Type -from envoy.base import utils +import abstracts -import utils as dep_utils +from envoy.dependency import cve_scan -# Subset of CVE fields that are useful below. -Cve = namedtuple( - 'Cve', - ['id', 'description', 'cpes', 'score', 'severity', 'published_date', 'last_modified_date']) +import tools.dependency.utils as dep_utils -class Cpe(namedtuple('CPE', ['part', 'vendor', 'product', 'version'])): - """Model a subset of CPE fields that are used in CPE matching.""" +@abstracts.implementer(cve_scan.ACVE) +class EnvoyCVE: - @classmethod - def from_string(cls, cpe_str): - assert (cpe_str.startswith('cpe:2.3:')) - components = cpe_str.split(':') - assert (len(components) >= 6) - return cls(*components[2:6]) + @property + def cpe_class(self): + return EnvoyCPE - def __str__(self): - return f'cpe:2.3:{self.part}:{self.vendor}:{self.product}:{self.version}' + @property + def version_matcher_class(self) -> Type[cve_scan.ACVEVersionMatcher]: + return EnvoyCVEVersionMatcher - def vendor_normalized(self): - """Return a normalized CPE where only part and vendor are significant.""" - return Cpe(self.part, self.vendor, '*', '*') +@abstracts.implementer(cve_scan.ACPE) +class EnvoyCPE: + pass -def parse_cve_json(cve_json, cves, cpe_revmap): - """Parse CVE JSON dictionary. - Args: - cve_json: a NIST CVE JSON dictionary. - cves: dictionary mapping CVE ID string to Cve object (output). - cpe_revmap: a reverse map from vendor normalized CPE to CVE ID string. - """ +@abstracts.implementer(cve_scan.ADependency) +class EnvoyDependency: + pass - # This provides an over-approximation of possible CPEs affected by CVE nodes - # metadata; it traverses the entire AND-OR tree and just gathers every CPE - # observed. Generally we expect that most of Envoy's CVE-CPE matches to be - # simple, plus it's interesting to consumers of this data to understand when a - # CPE pops up, even in a conditional setting. - def gather_cpes(nodes, cpe_set): - for node in nodes: - for cpe_match in node.get('cpe_match', []): - cpe_set.add(Cpe.from_string(cpe_match['cpe23Uri'])) - gather_cpes(node.get('children', []), cpe_set) - for cve in cve_json['CVE_Items']: - cve_id = cve['cve']['CVE_data_meta']['ID'] - description = cve['cve']['description']['description_data'][0]['value'] - cpe_set = set() - gather_cpes(cve['configurations']['nodes'], cpe_set) - if len(cpe_set) == 0: - continue +@abstracts.implementer(cve_scan.ACVEChecker) +class EnvoyCVEChecker: - if not "baseMetricV3" in cve['impact']: - print(f"WARNING: ignoring v2 metric for {cve['cve']['CVE_data_meta']['ID']}") - continue + @property + def cpe_class(self): + return EnvoyCPE - cvss_v3_score = cve['impact']['baseMetricV3']['cvssV3']['baseScore'] - cvss_v3_severity = cve['impact']['baseMetricV3']['cvssV3']['baseSeverity'] + @property + def cve_class(self): + return EnvoyCVE - def parse_cve_date(date_str): - assert (date_str.endswith('Z')) - return dt.date.fromisoformat(date_str.split('T')[0]) + @property + def dependency_class(self): + return EnvoyDependency - published_date = parse_cve_date(cve['publishedDate']) - last_modified_date = parse_cve_date(cve['lastModifiedDate']) - cves[cve_id] = Cve( - cve_id, description, cpe_set, cvss_v3_score, cvss_v3_severity, published_date, - last_modified_date) - for cpe in cpe_set: - cpe_revmap[str(cpe.vendor_normalized())].add(cve_id) - return cves, cpe_revmap + @cached_property + def dependency_metadata(self): + return dep_utils.repository_locations() + @cached_property + def ignored_cves(self): + return super().ignored_cves -def download_cve_data(urls): - """Download NIST CVE JSON databases from given URLs and parse. - Args: - urls: a list of URLs. - Returns: - cves: dictionary mapping CVE ID string to Cve object (output). - cpe_revmap: a reverse map from vendor normalized CPE to CVE ID string. - """ - cves = {} - cpe_revmap = defaultdict(set) - for url in urls: - print(f'Loading NIST CVE database from {url}...') - with urllib.request.urlopen(url) as request: - with gzip.GzipFile(fileobj=request) as json_data: - parse_cve_json(json.loads(json_data.read()), cves, cpe_revmap) - return cves, cpe_revmap +@abstracts.implementer(cve_scan.ACVEVersionMatcher) +class EnvoyCVEVersionMatcher: + pass -def format_cve_details(cve, deps): - formatted_deps = ', '.join(sorted(deps)) - wrapped_description = '\n '.join(textwrap.wrap(cve.description)) - return f""" - CVE ID: {cve.id} - CVSS v3 score: {cve.score} - Severity: {cve.severity} - Published date: {cve.published_date} - Last modified date: {cve.last_modified_date} - Dependencies: {formatted_deps} - Description: {wrapped_description} - Affected CPEs: - """ + '\n '.join(f'- {cpe}' for cpe in cve.cpes) +def main(*args) -> int: + return EnvoyCVEChecker(*args)() -FUZZY_DATE_RE = re.compile('(\d{4}).?(\d{2}).?(\d{2})') -FUZZY_SEMVER_RE = re.compile('(\d+)[:\.\-_](\d+)[:\.\-_](\d+)') - - -def regex_groups_match(regex, lhs, rhs): - """Do two strings match modulo a regular expression? - - Args: - regex: regular expression - lhs: LHS string - rhs: RHS string - Returns: - A boolean indicating match. - """ - lhs_match = regex.search(lhs) - if lhs_match: - rhs_match = regex.search(rhs) - if rhs_match and lhs_match.groups() == rhs_match.groups(): - return True - return False - - -def cpe_match(cpe, dep_metadata): - """Heuristically match dependency metadata against CPE. - - We have a number of rules below that should are easy to compute without having - to look at the dependency metadata. In the future, with additional access to - repository information we could do the following: - - For dependencies at a non-release version, walk back through git history to - the last known release version and attempt a match with this. - - For dependencies at a non-release version, use the commit date to look for a - version match where version is YYYY-MM-DD. - - Args: - cpe: Cpe object to match against. - dep_metadata: dependency metadata dictionary. - Returns: - A boolean indicating a match. - """ - dep_cpe = Cpe.from_string(dep_metadata['cpe']) - dep_version = dep_metadata['version'] - # The 'part' and 'vendor' must be an exact match. - if cpe.part != dep_cpe.part: - return False - if cpe.vendor != dep_cpe.vendor: - return False - # We allow Envoy dependency CPEs to wildcard the 'product', this is useful for - # LLVM where multiple product need to be covered. - if dep_cpe.product != '*' and cpe.product != dep_cpe.product: - return False - # Wildcard versions always match. - if cpe.version == '*': - return True - # An exact version match is a hit. - if cpe.version == dep_version: - return True - # Allow the 'release_date' dependency metadata to substitute for date. - # TODO(htuch): Consider fuzzier date ranges. - if cpe.version == dep_metadata['release_date']: - return True - # Try a fuzzy date match to deal with versions like fips-20190304 in dependency version. - if regex_groups_match(FUZZY_DATE_RE, dep_version, cpe.version): - return True - # Try a fuzzy semver match to deal with things like 2.1.0-beta3. - if regex_groups_match(FUZZY_SEMVER_RE, dep_version, cpe.version): - return True - # Fall-thru. - return False - - -def cve_match(cve, dep_metadata): - """Heuristically match dependency metadata against CVE. - - In general, we allow false positives but want to keep the noise low, to avoid - the toil around having to populate IGNORES_CVES. - - Args: - cve: Cve object to match against. - dep_metadata: dependency metadata dictionary. - Returns: - A boolean indicating a match. - """ - wildcard_version_match = False - # Consider each CPE attached to the CVE for a match against the dependency CPE. - for cpe in cve.cpes: - if cpe_match(cpe, dep_metadata): - # Wildcard version matches need additional heuristics unrelated to CPE to - # qualify, e.g. last updated date. - if cpe.version == '*': - wildcard_version_match = True - else: - return True - if wildcard_version_match: - # If the CVE was published after the dependency was last updated, it's a - # potential match. - last_dep_update = dt.date.fromisoformat(dep_metadata['release_date']) - if last_dep_update <= cve.published_date: - return True - return False - - -def cve_scan(cves, cpe_revmap, cve_allowlist, repository_locations): - """Scan for CVEs in a parsed NIST CVE database. - - Args: - cves: CVE dictionary as provided by download_cve_data(). - cve_revmap: CPE-CVE reverse map as provided by download_cve_data(). - cve_allowlist: an allowlist of CVE IDs to ignore. - repository_locations: a dictionary of dependency metadata in the format - described in api/bazel/external_deps.bzl. - Returns: - possible_cves: a dictionary mapping CVE IDs to Cve objects. - cve_deps: a dictionary mapping CVE IDs to dependency names. - """ - possible_cves = {} - cve_deps = defaultdict(list) - for dep, metadata in repository_locations.items(): - cpe = metadata.get('cpe', 'N/A') - if cpe == 'N/A': - continue - candidate_cve_ids = cpe_revmap.get(str(Cpe.from_string(cpe).vendor_normalized()), []) - for cve_id in candidate_cve_ids: - cve = cves[cve_id] - if cve.id in cve_allowlist: - continue - if cve_match(cve, metadata): - possible_cves[cve_id] = cve - cve_deps[cve_id].append(dep) - return possible_cves, cve_deps - - -if __name__ == '__main__': - cve_config = utils.from_yaml(sys.argv[1]) - - # Allow local overrides for NIST CVE database URLs via args. - urls = sys.argv[2:] - if not urls: - current_year = dt.datetime.now().year - scan_years = range(cve_config["start_year"], current_year + 1) - urls = [cve_config["ndist_url"].format(year=year) for year in scan_years] - cves, cpe_revmap = download_cve_data(urls) - - possible_cves, cve_deps = cve_scan( - cves, cpe_revmap, cve_config["ignore"], dep_utils.repository_locations()) - if possible_cves: - print( - '\nBased on heuristic matching with the NIST CVE database, Envoy may be vulnerable to:') - for cve_id in sorted(possible_cves): - print(f'{format_cve_details(possible_cves[cve_id], cve_deps[cve_id])}') - sys.exit(1) +if __name__ == "__main__": + sys.exit(main(*sys.argv[1:])) diff --git a/tools/dependency/cve_scan_test.py b/tools/dependency/cve_scan_test.py deleted file mode 100755 index 28ff3224c2f24..0000000000000 --- a/tools/dependency/cve_scan_test.py +++ /dev/null @@ -1,295 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for cve_scan.""" - -from collections import defaultdict -import datetime as dt -import unittest - -import cve_scan - - -class CveScanTest(unittest.TestCase): - - def test_parse_cve_json(self): - cve_json = { - 'CVE_Items': [ - { - 'cve': { - 'CVE_data_meta': { - 'ID': 'CVE-2020-1234' - }, - 'description': { - 'description_data': [{ - 'value': 'foo' - }] - } - }, - 'configurations': { - 'nodes': [{ - 'cpe_match': [{ - 'cpe23Uri': 'cpe:2.3:a:foo:bar:1.2.3' - }], - }], - }, - 'impact': { - 'baseMetricV3': { - 'cvssV3': { - 'baseScore': 3.4, - 'baseSeverity': 'LOW' - } - } - }, - 'publishedDate': '2020-03-17T00:59Z', - 'lastModifiedDate': '2020-04-17T00:59Z' - }, - { - 'cve': { - 'CVE_data_meta': { - 'ID': 'CVE-2020-1235' - }, - 'description': { - 'description_data': [{ - 'value': 'bar' - }] - } - }, - 'configurations': { - 'nodes': [{ - 'cpe_match': [{ - 'cpe23Uri': 'cpe:2.3:a:foo:bar:1.2.3' - }], - 'children': [ - { - 'cpe_match': [{ - 'cpe23Uri': 'cpe:2.3:a:foo:baz:3.2.3' - }] - }, - { - 'cpe_match': [{ - 'cpe23Uri': 'cpe:2.3:a:foo:*:*' - }, { - 'cpe23Uri': 'cpe:2.3:a:wat:bar:1.2.3' - }] - }, - ], - }], - }, - 'impact': { - 'baseMetricV3': { - 'cvssV3': { - 'baseScore': 9.9, - 'baseSeverity': 'HIGH' - } - } - }, - 'publishedDate': '2020-03-18T00:59Z', - 'lastModifiedDate': '2020-04-18T00:59Z' - }, - ] - } - cves = {} - cpe_revmap = defaultdict(set) - cve_scan.parse_cve_json(cve_json, cves, cpe_revmap) - self.maxDiff = None - self.assertDictEqual( - cves, { - 'CVE-2020-1234': - cve_scan.Cve( - id='CVE-2020-1234', - description='foo', - cpes=set([self.build_cpe('cpe:2.3:a:foo:bar:1.2.3')]), - score=3.4, - severity='LOW', - published_date=dt.date(2020, 3, 17), - last_modified_date=dt.date(2020, 4, 17)), - 'CVE-2020-1235': - cve_scan.Cve( - id='CVE-2020-1235', - description='bar', - cpes=set( - map( - self.build_cpe, [ - 'cpe:2.3:a:foo:bar:1.2.3', 'cpe:2.3:a:foo:baz:3.2.3', - 'cpe:2.3:a:foo:*:*', 'cpe:2.3:a:wat:bar:1.2.3' - ])), - score=9.9, - severity='HIGH', - published_date=dt.date(2020, 3, 18), - last_modified_date=dt.date(2020, 4, 18)) - }) - self.assertDictEqual( - cpe_revmap, { - 'cpe:2.3:a:foo:*:*': {'CVE-2020-1234', 'CVE-2020-1235'}, - 'cpe:2.3:a:wat:*:*': {'CVE-2020-1235'} - }) - - def build_cpe(self, cpe_str): - return cve_scan.Cpe.from_string(cpe_str) - - def build_dep(self, cpe_str, version=None, release_date=None): - return {'cpe': cpe_str, 'version': version, 'release_date': release_date} - - def cpe_match(self, cpe_str, dep_cpe_str, version=None, release_date=None): - return cve_scan.cpe_match( - self.build_cpe(cpe_str), - self.build_dep(dep_cpe_str, version=version, release_date=release_date)) - - def test_cpe_match(self): - # Mismatched part - self.assertFalse(self.cpe_match('cpe:2.3:o:foo:bar:*', 'cpe:2.3:a:foo:bar:*')) - # Mismatched vendor - self.assertFalse(self.cpe_match('cpe:2.3:a:foo:bar:*', 'cpe:2.3:a:foz:bar:*')) - # Mismatched product - self.assertFalse(self.cpe_match('cpe:2.3:a:foo:bar:*', 'cpe:2.3:a:foo:baz:*')) - # Wildcard product - self.assertTrue(self.cpe_match('cpe:2.3:a:foo:bar:*', 'cpe:2.3:a:foo:*:*')) - # Wildcard version match - self.assertTrue(self.cpe_match('cpe:2.3:a:foo:bar:*', 'cpe:2.3:a:foo:bar:*')) - # Exact version match - self.assertTrue( - self.cpe_match('cpe:2.3:a:foo:bar:1.2.3', 'cpe:2.3:a:foo:bar:*', version='1.2.3')) - # Date version match - self.assertTrue( - self.cpe_match( - 'cpe:2.3:a:foo:bar:2020-03-05', 'cpe:2.3:a:foo:bar:*', release_date='2020-03-05')) - fuzzy_version_matches = [ - ('2020-03-05', '2020-03-05'), - ('2020-03-05', '20200305'), - ('2020-03-05', 'foo-20200305-bar'), - ('2020-03-05', 'foo-2020_03_05-bar'), - ('2020-03-05', 'foo-2020-03-05-bar'), - ('1.2.3', '1.2.3'), - ('1.2.3', '1-2-3'), - ('1.2.3', '1_2_3'), - ('1.2.3', '1:2:3'), - ('1.2.3', 'foo-1-2-3-bar'), - ] - for cpe_version, dep_version in fuzzy_version_matches: - self.assertTrue( - self.cpe_match( - f'cpe:2.3:a:foo:bar:{cpe_version}', 'cpe:2.3:a:foo:bar:*', version=dep_version)) - fuzzy_version_no_matches = [ - ('2020-03-05', '2020-3.5'), - ('2020-03-05', '2020--03-05'), - ('1.2.3', '1@2@3'), - ('1.2.3', '1..2.3'), - ] - for cpe_version, dep_version in fuzzy_version_no_matches: - self.assertFalse( - self.cpe_match( - f'cpe:2.3:a:foo:bar:{cpe_version}', 'cpe:2.3:a:foo:bar:*', version=dep_version)) - - def build_cve(self, cve_id, cpes, published_date): - return cve_scan.Cve( - cve_id, - description=None, - cpes=cpes, - score=None, - severity=None, - published_date=dt.date.fromisoformat(published_date), - last_modified_date=None) - - def cve_match(self, cve_id, cpes, published_date, dep_cpe_str, version=None, release_date=None): - return cve_scan.cve_match( - self.build_cve(cve_id, cpes=cpes, published_date=published_date), - self.build_dep(dep_cpe_str, version=version, release_date=release_date)) - - def test_cve_match(self): - # Empty CPEs, no match - self.assertFalse(self.cve_match('CVE-2020-123', set(), '2020-05-03', 'cpe:2.3:a:foo:bar:*')) - # Wildcard version, stale dependency match - self.assertTrue( - self.cve_match( - 'CVE-2020-123', - set([self.build_cpe('cpe:2.3:a:foo:bar:*')]), - '2020-05-03', - 'cpe:2.3:a:foo:bar:*', - release_date='2020-05-02')) - self.assertTrue( - self.cve_match( - 'CVE-2020-123', - set([self.build_cpe('cpe:2.3:a:foo:bar:*')]), - '2020-05-03', - 'cpe:2.3:a:foo:bar:*', - release_date='2020-05-03')) - # Wildcard version, recently updated - self.assertFalse( - self.cve_match( - 'CVE-2020-123', - set([self.build_cpe('cpe:2.3:a:foo:bar:*')]), - '2020-05-03', - 'cpe:2.3:a:foo:bar:*', - release_date='2020-05-04')) - # Version match - self.assertTrue( - self.cve_match( - 'CVE-2020-123', - set([self.build_cpe('cpe:2.3:a:foo:bar:1.2.3')]), - '2020-05-03', - 'cpe:2.3:a:foo:bar:*', - version='1.2.3')) - # Version mismatch - self.assertFalse( - self.cve_match( - 'CVE-2020-123', - set([self.build_cpe('cpe:2.3:a:foo:bar:1.2.3')]), - '2020-05-03', - 'cpe:2.3:a:foo:bar:*', - version='1.2.4', - release_date='2020-05-02')) - # Multiple CPEs, match first, don't match later. - self.assertTrue( - self.cve_match( - 'CVE-2020-123', - set([ - self.build_cpe('cpe:2.3:a:foo:bar:1.2.3'), - self.build_cpe('cpe:2.3:a:foo:baz:3.2.1') - ]), - '2020-05-03', - 'cpe:2.3:a:foo:bar:*', - version='1.2.3')) - - def test_cve_scan(self): - cves = { - 'CVE-2020-1234': - self.build_cve( - 'CVE-2020-1234', - set([ - self.build_cpe('cpe:2.3:a:foo:bar:1.2.3'), - self.build_cpe('cpe:2.3:a:foo:baz:3.2.1') - ]), '2020-05-03'), - 'CVE-2020-1235': - self.build_cve( - 'CVE-2020-1235', - set([ - self.build_cpe('cpe:2.3:a:foo:bar:1.2.3'), - self.build_cpe('cpe:2.3:a:foo:baz:3.2.1') - ]), '2020-05-03'), - 'CVE-2020-1236': - self.build_cve( - 'CVE-2020-1236', set([ - self.build_cpe('cpe:2.3:a:foo:wat:1.2.3'), - ]), '2020-05-03'), - } - cpe_revmap = { - 'cpe:2.3:a:foo:*:*': ['CVE-2020-1234', 'CVE-2020-1235', 'CVE-2020-1236'], - } - cve_allowlist = ['CVE-2020-1235'] - repository_locations = { - 'bar': self.build_dep('cpe:2.3:a:foo:bar:*', version='1.2.3'), - 'baz': self.build_dep('cpe:2.3:a:foo:baz:*', version='3.2.1'), - 'foo': self.build_dep('cpe:2.3:a:foo:*:*', version='1.2.3'), - 'blah': self.build_dep('N/A'), - } - possible_cves, cve_deps = cve_scan.cve_scan( - cves, cpe_revmap, cve_allowlist, repository_locations) - self.assertListEqual(sorted(possible_cves.keys()), ['CVE-2020-1234', 'CVE-2020-1236']) - self.assertDictEqual( - cve_deps, { - 'CVE-2020-1234': ['bar', 'baz', 'foo'], - 'CVE-2020-1236': ['foo'] - }) - - -if __name__ == '__main__': - unittest.main() From 1143dd703229843f12a4ed0e12c85b39bea0cf75 Mon Sep 17 00:00:00 2001 From: phlax Date: Mon, 29 Nov 2021 15:50:33 +0000 Subject: [PATCH 25/51] tools: Fix dependency checker release dates bug (#19109) Signed-off-by: Ryan Northey --- tools/dependency/release_dates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dependency/release_dates.py b/tools/dependency/release_dates.py index aed1021071898..3cf62b7129c66 100644 --- a/tools/dependency/release_dates.py +++ b/tools/dependency/release_dates.py @@ -211,7 +211,7 @@ def get_tagged_release_date(repo, metadata_version, github_release): latest = '' print(f'GithubException {repo.name}: {err.data} {err.status} while getting latest release.') - if latest and github_release.version <= latest.tag_name: + if latest: release = repo.get_release(github_release.version) return release.published_at else: From 209b7ba418db99d20ed60e5238e3269f4c7a6591 Mon Sep 17 00:00:00 2001 From: Ryan Hamilton Date: Mon, 29 Nov 2021 09:19:10 -0800 Subject: [PATCH 26/51] Update QUICHE from c2ddf95dc to 7f2d442e3 (#19095) https://github.com/google/quiche/compare/c2ddf95dc..7f2d442e3 $ git log c2ddf95dc..7f2d442e3 --date=short --no-merges --format="%ad %al %s" 2021-11-24 wub Replace --gfe2_reloadable_flag_quic_add_cached_network_parameters_to_address_token by --gfe2_reloadable_flag_quic_add_cached_network_parameters_to_address_token2. 2021-11-23 fayang Internal change Risk Level: Low Testing: Unit Tests Docs Changes: N/A Release Notes: N/A Platform Specific Features: N/A Signed-off-by: Ryan Hamilton --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 16207ad99e713..6d4bac9928ee3 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -828,12 +828,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "QUICHE", project_desc = "QUICHE (QUIC, HTTP/2, Etc) is Google‘s implementation of QUIC and related protocols", project_url = "https://github.com/google/quiche", - version = "c2ddf95dc2380e1d7cd5aa8fe9f61b4e01b23e6b", - sha256 = "435dc3c3858d8328c48fb633bc46431aa5aea2a29e0b5f125f743b0b5b5c1513", + version = "7f2d442e3cb02b4ef4892e62b0e9a8ce94a83db2", + sha256 = "7de89aa92bb23b66b130891dc4d73b3aa1514271d39d261239fe7eef7744166d", urls = ["https://github.com/google/quiche/archive/{version}.tar.gz"], strip_prefix = "quiche-{version}", use_category = ["dataplane_core"], - release_date = "2021-11-22", + release_date = "2021-11-24", cpe = "N/A", ), com_googlesource_googleurl = dict( From 3b45d6cf465b8ce3c268edaa5c70722811dbb39f Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Mon, 29 Nov 2021 10:08:36 -0800 Subject: [PATCH 27/51] Overload: Reset H2 server stream only use codec level reset mechanism (#18895) Signed-off-by: Kevin Baichoo --- source/common/http/conn_manager_impl.cc | 7 ++ source/common/http/http2/codec_impl.cc | 7 +- test/common/http/conn_manager_impl_test_2.cc | 87 ++++++++++--------- .../http/conn_manager_impl_test_base.cc | 4 +- .../buffer_accounting_integration_test.cc | 87 +++++++++++++++++++ test/mocks/http/stream.cc | 17 +++- test/mocks/http/stream.h | 12 ++- 7 files changed, 170 insertions(+), 51 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index d9c654a596339..d39aa9bda4b3a 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -260,6 +260,13 @@ void ConnectionManagerImpl::doDeferredStreamDestroy(ActiveStream& stream) { read_callbacks_->connection().dispatcher().deferredDelete(stream.removeFromList(streams_)); + // The response_encoder should never be dangling (unless we're destroying a + // stream we are recreating) as the codec level stream will either outlive the + // ActiveStream, or be alive in deferred deletion queue at this point. + if (stream.response_encoder_) { + stream.response_encoder_->getStream().removeCallbacks(stream); + } + if (connection_idle_timer_ && streams_.empty()) { connection_idle_timer_->enableTimer(config_.idleTimeout().value()); } diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 1e19b3c752904..c681acb6525df 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -639,8 +639,11 @@ void ConnectionImpl::StreamImpl::resetStream(StreamResetReason reason) { // If we submit a reset, nghttp2 will cancel outbound frames that have not yet been sent. // We want these frames to go out so we defer the reset until we send all of the frames that - // end the local stream. - if (useDeferredReset() && local_end_stream_ && !local_end_stream_sent_) { + // end the local stream. However, if we're resetting the stream due to + // overload, we should reset the stream as soon as possible to free used + // resources. + if (useDeferredReset() && local_end_stream_ && !local_end_stream_sent_ && + reason != StreamResetReason::OverloadManager) { ASSERT(parent_.getStream(stream_id_) != nullptr); parent_.pending_deferred_reset_streams_.emplace(stream_id_, this); deferred_reset_ = reason; diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index 44bd0db57ddec..42f279490a7da 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -150,7 +150,7 @@ TEST_F(HttpConnectionManagerImplTest, DownstreamProtocolError) { return codecProtocolError("protocol error"); })); - EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)); + EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)).Times(2); EXPECT_CALL(filter_factory_, createFilterChain(_)).Times(0); // A protocol exception should result in reset of the streams followed by a remote or local close @@ -232,7 +232,7 @@ TEST_F(HttpConnectionManagerImplTest, FrameFloodError) { return bufferFloodError("too many outbound frames"); })); - EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)); + EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)).Times(2); EXPECT_CALL(filter_factory_, createFilterChain(_)).Times(0); // FrameFloodException should result in reset of the streams followed by abortive close. @@ -1104,7 +1104,7 @@ TEST_F(HttpConnectionManagerImplTest, UpstreamWatermarkCallbacks) { EXPECT_EQ(1U, stats_.named_.downstream_flow_control_resumed_reading_total_.value()); // Backup upstream once again. - EXPECT_CALL(response_encoder_, getStream()).WillOnce(ReturnRef(stream_)); + EXPECT_CALL(response_encoder_, getStream()).WillRepeatedly(ReturnRef(stream_)); EXPECT_CALL(stream_, readDisable(true)); ASSERT(decoder_filters_[0]->callbacks_ != nullptr); decoder_filters_[0]->callbacks_->onDecoderFilterAboveWriteBufferHighWatermark(); @@ -1334,7 +1334,7 @@ TEST_F(HttpConnectionManagerImplTest, HitFilterWatermarkLimits) { })); expectOnDestroy(); - EXPECT_CALL(stream_, removeCallbacks(_)); + EXPECT_CALL(stream_, removeCallbacks(_)).Times(2); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::LocalClose); } @@ -1364,45 +1364,47 @@ TEST_F(HttpConnectionManagerImplTest, HitRequestBufferLimits) { // Return 413 from an intermediate filter and make sure we don't continue the filter chain. TEST_F(HttpConnectionManagerImplTest, HitRequestBufferLimitsIntermediateFilter) { - InSequence s; - initial_buffer_limit_ = 10; - setup(false, ""); - - EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> Http::Status { - decoder_ = &conn_manager_->newStream(response_encoder_); - RequestHeaderMapPtr headers{ - new TestRequestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; - decoder_->decodeHeaders(std::move(headers), false); + { + InSequence s; + initial_buffer_limit_ = 10; + setup(false, ""); - Buffer::OwnedImpl fake_data("hello"); - decoder_->decodeData(fake_data, false); + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> Http::Status { + decoder_ = &conn_manager_->newStream(response_encoder_); + RequestHeaderMapPtr headers{ + new TestRequestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; + decoder_->decodeHeaders(std::move(headers), false); - Buffer::OwnedImpl fake_data2("world world"); - decoder_->decodeData(fake_data2, true); - return Http::okStatus(); - })); + Buffer::OwnedImpl fake_data("hello"); + decoder_->decodeData(fake_data, false); - setUpBufferLimits(); - setupFilterChain(2, 1); + Buffer::OwnedImpl fake_data2("world world"); + decoder_->decodeData(fake_data2, true); + return Http::okStatus(); + })); - EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, false)) - .WillOnce(Return(FilterHeadersStatus::StopIteration)); - EXPECT_CALL(*decoder_filters_[0], decodeData(_, false)) - .WillOnce(Return(FilterDataStatus::StopIterationAndBuffer)); - EXPECT_CALL(*decoder_filters_[0], decodeData(_, true)) - .WillOnce(Return(FilterDataStatus::Continue)); - EXPECT_CALL(*decoder_filters_[0], decodeComplete()); - Http::TestResponseHeaderMapImpl response_headers{ - {":status", "413"}, {"content-length", "17"}, {"content-type", "text/plain"}}; - EXPECT_CALL(*encoder_filters_[0], encodeHeaders(HeaderMapEqualRef(&response_headers), false)) - .WillOnce(Return(FilterHeadersStatus::StopIteration)); - EXPECT_CALL(*encoder_filters_[0], encodeData(_, true)) - .WillOnce(Return(FilterDataStatus::StopIterationAndWatermark)); - EXPECT_CALL(*encoder_filters_[0], encodeComplete()); + setUpBufferLimits(); + setupFilterChain(2, 1); - // Kick off the incoming data. - Buffer::OwnedImpl fake_input("1234"); - conn_manager_->onData(fake_input, false); + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + EXPECT_CALL(*decoder_filters_[0], decodeData(_, false)) + .WillOnce(Return(FilterDataStatus::StopIterationAndBuffer)); + EXPECT_CALL(*decoder_filters_[0], decodeData(_, true)) + .WillOnce(Return(FilterDataStatus::Continue)); + EXPECT_CALL(*decoder_filters_[0], decodeComplete()); + Http::TestResponseHeaderMapImpl response_headers{ + {":status", "413"}, {"content-length", "17"}, {"content-type", "text/plain"}}; + EXPECT_CALL(*encoder_filters_[0], encodeHeaders(HeaderMapEqualRef(&response_headers), false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + EXPECT_CALL(*encoder_filters_[0], encodeData(_, true)) + .WillOnce(Return(FilterDataStatus::StopIterationAndWatermark)); + EXPECT_CALL(*encoder_filters_[0], encodeComplete()); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); + } doRemoteClose(false); } @@ -1469,17 +1471,20 @@ TEST_F(HttpConnectionManagerImplTest, HitResponseBufferLimitsAfterHeaders) { const std::string data = "A long enough string to go over watermarks"; Buffer::OwnedImpl fake_response(data); InSequence s; - EXPECT_CALL(stream_, removeCallbacks(_)); - expectOnDestroy(false); EXPECT_CALL(*encoder_filters_[1], encodeData(_, false)) .WillOnce(Return(FilterDataStatus::StopIterationAndBuffer)); EXPECT_CALL(stream_, resetStream(_)); - filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::LocalClose); EXPECT_LOG_CONTAINS( "debug", "Resetting stream due to response_payload_too_large. Prior headers have already been sent", decoder_filters_[0]->callbacks_->encodeData(fake_response, false);); EXPECT_EQ(1U, stats_.named_.rs_too_large_.value()); + + // Clean up connection + EXPECT_CALL(stream_, removeCallbacks(_)); + expectOnDestroy(false); + EXPECT_CALL(stream_, removeCallbacks(_)); + filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::LocalClose); } TEST_F(HttpConnectionManagerImplTest, FilterHeadReply) { diff --git a/test/common/http/conn_manager_impl_test_base.cc b/test/common/http/conn_manager_impl_test_base.cc index 0ecfafee44fd1..14ba94dd1a031 100644 --- a/test/common/http/conn_manager_impl_test_base.cc +++ b/test/common/http/conn_manager_impl_test_base.cc @@ -264,7 +264,9 @@ void HttpConnectionManagerImplTest::expectOnDestroy(bool deferred) { } void HttpConnectionManagerImplTest::doRemoteClose(bool deferred) { - EXPECT_CALL(stream_, removeCallbacks(_)); + // We will call removeCallbacks twice. + // Once in resetAllStreams, and once in doDeferredStreamDestroy. + EXPECT_CALL(stream_, removeCallbacks(_)).Times(2); expectOnDestroy(deferred); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } diff --git a/test/integration/buffer_accounting_integration_test.cc b/test/integration/buffer_accounting_integration_test.cc index 391cf60ec8615..13f68cc08a40f 100644 --- a/test/integration/buffer_accounting_integration_test.cc +++ b/test/integration/buffer_accounting_integration_test.cc @@ -23,6 +23,8 @@ namespace Envoy { namespace { +using testing::HasSubstr; + std::string protocolTestParamsAndBoolToString( const ::testing::TestParamInfo>& params) { return fmt::format("{}_{}_{}", @@ -617,4 +619,89 @@ TEST_P(Http2OverloadManagerIntegrationTest, EXPECT_EQ(smallest_response->headers().getStatusValue(), "200"); } +TEST_P(Http2OverloadManagerIntegrationTest, CanResetStreamIfEnvoyLevelStreamEnded) { + useAccessLog("%RESPONSE_CODE%"); + initializeOverloadManagerInBootstrap( + TestUtility::parseYaml(R"EOF( + name: "envoy.overload_actions.reset_high_memory_stream" + triggers: + - name: "envoy.resource_monitors.testonly.fake_resource_monitor" + scaled: + scaling_threshold: 0.90 + saturation_threshold: 0.98 + )EOF")); + initialize(); + + // Set 10MiB receive window for the client. + const int downstream_window_size = 10 * 1024 * 1024; + envoy::config::core::v3::Http2ProtocolOptions http2_options = + ::Envoy::Http2::Utility::initializeAndValidateOptions( + envoy::config::core::v3::Http2ProtocolOptions()); + http2_options.mutable_initial_stream_window_size()->set_value(downstream_window_size); + http2_options.mutable_initial_connection_window_size()->set_value(downstream_window_size); + codec_client_ = makeRawHttpConnection(makeClientConnection(lookupPort("http")), http2_options); + + // Makes us have Envoy's writes to downstream return EAGAIN + writev_matcher_->setSourcePort(lookupPort("http")); + writev_matcher_->setWritevReturnsEgain(); + + // Send a request + auto encoder_decoder = codec_client_->startRequest(Http::TestRequestHeaderMapImpl{ + {":method", "POST"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"content-length", "10"}, + }); + auto& encoder = encoder_decoder.first; + const std::string data(10, 'a'); + codec_client_->sendData(encoder, data, true); + auto response = std::move(encoder_decoder.second); + + waitForNextUpstreamRequest(); + FakeStreamPtr upstream_request_for_response = std::move(upstream_request_); + + // Send the responses back. It is larger than the downstream's receive window + // size. Thus, the codec will not end the stream, but the Envoy level stream + // should. + upstream_request_for_response->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, + false); + const int response_size = downstream_window_size + 1024; // Slightly over the window size. + upstream_request_for_response->encodeData(response_size, true); + + if (streamBufferAccounting()) { + // Wait for access log to know the Envoy level stream has been deleted. + EXPECT_THAT(waitForAccessLog(access_log_name_), HasSubstr("200")); + } + + // Set the pressure so the overload action kills the response if doing stream + // accounting + updateResource(0.95); + test_server_->waitForGaugeEq( + "overload.envoy.overload_actions.reset_high_memory_stream.scale_percent", 62); + + if (streamBufferAccounting()) { + test_server_->waitForCounterGe("envoy.overload_actions.reset_high_memory_stream.count", 1); + } + + // Reduce resource pressure + updateResource(0.80); + test_server_->waitForGaugeEq( + "overload.envoy.overload_actions.reset_high_memory_stream.scale_percent", 0); + + // Resume writes to downstream. + writev_matcher_->setResumeWrites(); + + if (streamBufferAccounting()) { + EXPECT_TRUE(response->waitForReset()); + EXPECT_TRUE(response->reset()); + } else { + // If we're not doing the accounting, we didn't end up resetting the + // streams. + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ(response->headers().getStatusValue(), "200"); + } +} + } // namespace Envoy diff --git a/test/mocks/http/stream.cc b/test/mocks/http/stream.cc index 19181d8c26ed6..9410afdf2774c 100644 --- a/test/mocks/http/stream.cc +++ b/test/mocks/http/stream.cc @@ -13,12 +13,21 @@ MockStream::MockStream() { })); ON_CALL(*this, removeCallbacks(_)) - .WillByDefault( - Invoke([this](StreamCallbacks& callbacks) -> void { callbacks_.remove(&callbacks); })); + .WillByDefault(Invoke([this](StreamCallbacks& callbacks) -> void { + for (auto& callback : callbacks_) { + if (callback == &callbacks) { + callback = nullptr; + return; + } + } + })); ON_CALL(*this, resetStream(_)).WillByDefault(Invoke([this](StreamResetReason reason) -> void { - for (StreamCallbacks* callbacks : callbacks_) { - callbacks->onResetStream(reason, absl::string_view()); + for (auto& callback : callbacks_) { + if (callback) { + callback->onResetStream(reason, absl::string_view()); + callback = nullptr; + } } })); diff --git a/test/mocks/http/stream.h b/test/mocks/http/stream.h index 9a0abc4c58df5..a9d27e58ddefa 100644 --- a/test/mocks/http/stream.h +++ b/test/mocks/http/stream.h @@ -23,19 +23,25 @@ class MockStream : public Stream { MOCK_METHOD(void, setFlushTimeout, (std::chrono::milliseconds timeout)); MOCK_METHOD(void, setAccount, (Buffer::BufferMemoryAccountSharedPtr)); - std::list callbacks_{}; + // Use the same underlying structure as StreamCallbackHelper to insure iteration stability + // if we remove callbacks during iteration. + absl::InlinedVector callbacks_; Network::Address::InstanceConstSharedPtr connection_local_address_; Buffer::BufferMemoryAccountSharedPtr account_; void runHighWatermarkCallbacks() { for (auto* callback : callbacks_) { - callback->onAboveWriteBufferHighWatermark(); + if (callback) { + callback->onAboveWriteBufferHighWatermark(); + } } } void runLowWatermarkCallbacks() { for (auto* callback : callbacks_) { - callback->onBelowWriteBufferLowWatermark(); + if (callback) { + callback->onBelowWriteBufferLowWatermark(); + } } } From b432368ad4085655b840d5dc6942256cfafd3e0e Mon Sep 17 00:00:00 2001 From: Yao Zengzeng Date: Tue, 30 Nov 2021 03:45:48 +0800 Subject: [PATCH 28/51] http2: drain only once when reached max_requests_per_connection (#19078) Commit Message: drain only once when reached max_requests_per_connection Additional Description: fixes #19045 Risk Level: low Testing: unit test Docs Changes: n/a Signed-off-by: YaoZengzeng --- docs/root/version_history/current.rst | 1 + source/common/http/conn_manager_impl.cc | 2 +- test/common/http/conn_manager_impl_test_2.cc | 29 +++++++++++++++++++ .../common/http/conn_manager_impl_test_base.h | 3 +- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 0830081fae39c..1e4b8890da667 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -17,6 +17,7 @@ Minor Behavior Changes * dns: now respecting the returned DNS TTL for resolved hosts, rather than always relying on the hard-coded :ref:`dns_refresh_rate. ` This behavior can be temporarily reverted by setting the runtime guard ``envoy.reloadable_features.use_dns_ttl`` to false. * http: envoy will now proxy 102 and 103 headers from upstream, though as with 100s only the first 1xx response headers will be sent. This behavioral change by can temporarily reverted by setting runtime guard ``envoy.reloadable_features.proxy_102_103`` to false. * http: usage of the experimental matching API is no longer guarded behind a feature flag, as the corresponding protobuf fields have been marked as WIP. +* http: when envoy run out of ``max_requests_per_connection``, it will send an HTTP/2 "shutdown nofitication" (GOAWAY frame with max stream ID) and go to a default grace period of 5000 milliseconds (5 seconds) if ``drain_timeout`` is not specified. During this grace period, envoy will continue to accept new streams. After the grace period, a final GOAWAY is sent and envoy will start refusing new streams. However before bugfix, during the grace period, every time a new stream is received, old envoy will always send a "shutdown notification" and restart drain again which actually causes the grace period to be extended and is no longer equal to ``drain_timeout``. * json: switching from rapidjson to nlohmann/json. This behavioral change can be temporarily reverted by setting runtime guard ``envoy.reloadable_features.remove_legacy_json`` to false. * listener: destroy per network filter chain stats when a network filter chain is removed during the listener in place update. * quic: add back the support for IETF draft 29 which is guarded via ``envoy.reloadable_features.FLAGS_quic_reloadable_flag_quic_disable_version_draft_29``. It is off by default so Envoy only supports RFCv1 without flipping this runtime guard explicitly. Draft 29 is not recommended for use. diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index d39aa9bda4b3a..d1ab035b80505 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -296,7 +296,7 @@ RequestDecoder& ConnectionManagerImpl::newStream(ResponseEncoder& response_encod new_stream->state_.saw_connection_close_ = true; // Prevent erroneous debug log of closing due to incoming connection close header. drain_state_ = DrainState::Closing; - } else { + } else if (drain_state_ == DrainState::NotDraining) { startDrainSequence(); } ENVOY_CONN_LOG(debug, "max requests per connection reached", read_callbacks_->connection()); diff --git a/test/common/http/conn_manager_impl_test_2.cc b/test/common/http/conn_manager_impl_test_2.cc index 42f279490a7da..5f6db2acca6cd 100644 --- a/test/common/http/conn_manager_impl_test_2.cc +++ b/test/common/http/conn_manager_impl_test_2.cc @@ -349,6 +349,35 @@ TEST_F(HttpConnectionManagerImplTest, ConnectionDurationNoCodec) { EXPECT_EQ(1U, stats_.named_.downstream_cx_max_duration_reached_.value()); } +// Regression test for https://github.com/envoyproxy/envoy/issues/19045 +TEST_F(HttpConnectionManagerImplTest, MaxRequests) { + max_requests_per_connection_ = 1; + codec_->protocol_ = Protocol::Http2; + setup(false, ""); + + Event::MockTimer* drain_timer = setUpTimer(); + EXPECT_CALL(*drain_timer, enableTimer(_, _)); + + EXPECT_CALL(*codec_, dispatch(_)).WillRepeatedly(Invoke([&](Buffer::Instance&) -> Http::Status { + conn_manager_->newStream(response_encoder_); + return Http::okStatus(); + })); + + EXPECT_CALL(*codec_, goAway()); + EXPECT_CALL(*codec_, shutdownNotice()); + EXPECT_CALL(*drain_timer, disableTimer()); + + // Kick off two requests. + Buffer::OwnedImpl fake_input("hello"); + conn_manager_->onData(fake_input, false); + conn_manager_->onData(fake_input, false); + drain_timer->invokeCallback(); + + EXPECT_EQ(2U, stats_.named_.downstream_cx_max_requests_reached_.value()); + + conn_manager_->onEvent(Network::ConnectionEvent::RemoteClose); +} + TEST_F(HttpConnectionManagerImplTest, ConnectionDuration) { max_connection_duration_ = (std::chrono::milliseconds(10)); Event::MockTimer* connection_duration_timer = setUpTimer(); diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index 6225b61d854d2..f3a2c0eda8b2f 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -153,7 +153,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan originalIpDetectionExtensions() const override { return ip_detection_extensions_; } - uint64_t maxRequestsPerConnection() const override { return 0; } + uint64_t maxRequestsPerConnection() const override { return max_requests_per_connection_; } Envoy::Event::SimulatedTimeSystem test_time_; NiceMock route_config_provider_; @@ -184,6 +184,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan absl::optional user_agent_; uint32_t max_request_headers_kb_{Http::DEFAULT_MAX_REQUEST_HEADERS_KB}; uint32_t max_request_headers_count_{Http::DEFAULT_MAX_HEADERS_COUNT}; + uint64_t max_requests_per_connection_{}; absl::optional idle_timeout_; absl::optional max_connection_duration_; std::chrono::milliseconds stream_idle_timeout_{}; From bf324a70e44bf0e6e1a6f8f183ba964657948794 Mon Sep 17 00:00:00 2001 From: Faseela K Date: Tue, 30 Nov 2021 12:48:01 +0100 Subject: [PATCH 29/51] Fix verify_and_print_latest_release logic (#19111) Fixes #19109 Fix #19136 Signed-off-by: Faseela K --- tools/dependency/release_dates.py | 52 ++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/tools/dependency/release_dates.py b/tools/dependency/release_dates.py index 3cf62b7129c66..013d7f51ada34 100644 --- a/tools/dependency/release_dates.py +++ b/tools/dependency/release_dates.py @@ -24,6 +24,7 @@ import utils from colorama import Fore, Style from packaging import version +from packaging.version import parse as parse_version # Tag issues created with these labels. LABELS = ['dependencies', 'area/build', 'no stalebot'] @@ -60,16 +61,31 @@ def format_utc_date(date): return date.date().isoformat() +# Get the chronologically latest release from a github repo +def get_latest_release(repo, version_min): + current_version = parse_version(version_min) + latest_version = current_version + latest_release = None + for release in repo.get_releases(): + version = parse_version(release.tag_name) + if not version: + continue + if version >= latest_version: + latest_release = release + latest_version = version + return latest_release + + # Obtain latest release version and compare against metadata version, warn on # mismatch. def verify_and_print_latest_release(dep, repo, metadata_version, release_date, create_issue=False): try: - latest_release = repo.get_latest_release() + latest_release = get_latest_release(repo, metadata_version) except github.GithubException as err: # Repositories can not have releases or if they have releases may not publish a latest releases. Return print(f'GithubException {repo.name}: {err.data} {err.status} while getting latest release.') return - if latest_release.created_at > release_date and latest_release.tag_name != metadata_version: + if latest_release and latest_release.created_at > release_date and latest_release.tag_name != metadata_version: print( f'{Fore.YELLOW}*WARNING* {dep} has a newer release than {metadata_version}@<{release_date}>: ' f'{latest_release.tag_name}@<{latest_release.created_at}>{Style.RESET_ALL}') @@ -203,29 +219,27 @@ def verify_and_print_release_date(dep, github_release_date, metadata_release_dat # Extract release date from GitHub API for tagged releases. def get_tagged_release_date(repo, metadata_version, github_release): - try: - latest = repo.get_latest_release() + latest = get_latest_release(repo, github_release.version) + if latest: + release = repo.get_release(github_release.version) + return release.published_at except github.GithubException as err: # Repositories can not have releases or if they have releases may not publish a latest releases. If this is the case we keep going latest = '' print(f'GithubException {repo.name}: {err.data} {err.status} while getting latest release.') - if latest: - release = repo.get_release(github_release.version) - return release.published_at - else: - tags = repo.get_tags() - current_metadata_tag_commit_date = '' - for tag in tags.reversed: - if tag.name == github_release.version: - current_metadata_tag_commit_date = tag.commit.commit.committer.date - if not version.parse(tag.name).is_prerelease and version.parse( - tag.name) > version.parse(github_release.version): - print( - f'{Fore.YELLOW}*WARNING* {repo.name} has a newer release than {github_release.version}@<{current_metadata_tag_commit_date}>: ' - f'{tag.name}@<{tag.commit.commit.committer.date}>{Style.RESET_ALL}') - return current_metadata_tag_commit_date + tags = repo.get_tags() + current_metadata_tag_commit_date = '' + for tag in tags.reversed: + if tag.name == github_release.version: + current_metadata_tag_commit_date = tag.commit.commit.committer.date + if not version.parse(tag.name).is_prerelease and version.parse(tag.name) > version.parse( + github_release.version): + print( + f'{Fore.YELLOW}*WARNING* {repo.name} has a newer release than {github_release.version}@<{current_metadata_tag_commit_date}>: ' + f'{tag.name}@<{tag.commit.commit.committer.date}>{Style.RESET_ALL}') + return current_metadata_tag_commit_date # Extract release date from GitHub API for untagged releases. From d1fc2e3fa32966ddb31555f712c9895353e301d7 Mon Sep 17 00:00:00 2001 From: StarryNight Date: Tue, 30 Nov 2021 22:35:55 +0800 Subject: [PATCH 30/51] lua: support body setBytes with header content length set automatically (#18989) Signed-off-by: wangkai19 --- .../http/http_filters/lua_filter.rst | 6 ++--- .../extensions/filters/common/lua/wrappers.cc | 1 + .../extensions/filters/common/lua/wrappers.h | 4 +++- .../extensions/filters/http/lua/lua_filter.cc | 23 ++++++++++--------- .../extensions/filters/http/lua/lua_filter.h | 12 +++++----- .../filters/common/lua/wrappers_test.cc | 6 +++-- .../filters/http/lua/lua_integration_test.cc | 16 +++++++++++++ 7 files changed, 44 insertions(+), 24 deletions(-) diff --git a/docs/root/configuration/http/http_filters/lua_filter.rst b/docs/root/configuration/http/http_filters/lua_filter.rst index 4e55541f9a05a..28dad9499db64 100644 --- a/docs/root/configuration/http/http_filters/lua_filter.rst +++ b/docs/root/configuration/http/http_filters/lua_filter.rst @@ -248,8 +248,7 @@ There are two ways of doing this, the first one is via the ``body()`` API. .. code-block:: lua function envoy_on_response(response_handle) - local content_length = response_handle:body():setBytes("Not Found") - response_handle:headers():replace("content-length", content_length) + response_handle:body():setBytes("Not Found") response_handle:headers():replace("content-type", "text/html") end @@ -260,8 +259,7 @@ Or, through ``bodyChunks()`` API, which let Envoy to skip buffering the upstream function envoy_on_response(response_handle) - -- Sets the content-length. - response_handle:headers():replace("content-length", 28) + -- Sets the content-type. response_handle:headers():replace("content-type", "text/html") local last diff --git a/source/extensions/filters/common/lua/wrappers.cc b/source/extensions/filters/common/lua/wrappers.cc index bfb2a1c473259..9963a446cb773 100644 --- a/source/extensions/filters/common/lua/wrappers.cc +++ b/source/extensions/filters/common/lua/wrappers.cc @@ -68,6 +68,7 @@ int BufferWrapper::luaSetBytes(lua_State* state) { data_.drain(data_.length()); absl::string_view bytes = getStringViewFromLuaString(state, 2); data_.add(bytes); + headers_.setContentLength(data_.length()); lua_pushnumber(state, data_.length()); return 1; } diff --git a/source/extensions/filters/common/lua/wrappers.h b/source/extensions/filters/common/lua/wrappers.h index 344467d39ff2c..391120bd3f45f 100644 --- a/source/extensions/filters/common/lua/wrappers.h +++ b/source/extensions/filters/common/lua/wrappers.h @@ -16,7 +16,8 @@ namespace Lua { */ class BufferWrapper : public BaseLuaObject { public: - BufferWrapper(Buffer::Instance& data) : data_(data) {} + BufferWrapper(Http::RequestOrResponseHeaderMap& headers, Buffer::Instance& data) + : data_(data), headers_(headers) {} static ExportedFunctions exportedFunctions() { return {{"length", static_luaLength}, @@ -46,6 +47,7 @@ class BufferWrapper : public BaseLuaObject { DECLARE_LUA_FUNCTION(BufferWrapper, luaSetBytes); Buffer::Instance& data_; + Http::RequestOrResponseHeaderMap& headers_; }; class MetadataMapWrapper; diff --git a/source/extensions/filters/http/lua/lua_filter.cc b/source/extensions/filters/http/lua/lua_filter.cc index 9be9dd3d49b4a..7442154d0aad9 100644 --- a/source/extensions/filters/http/lua/lua_filter.cc +++ b/source/extensions/filters/http/lua/lua_filter.cc @@ -191,8 +191,9 @@ PerLuaCodeSetup::PerLuaCodeSetup(const std::string& lua_code, ThreadLocal::SlotA } StreamHandleWrapper::StreamHandleWrapper(Filters::Common::Lua::Coroutine& coroutine, - Http::HeaderMap& headers, bool end_stream, Filter& filter, - FilterCallbacks& callbacks, TimeSource& time_source) + Http::RequestOrResponseHeaderMap& headers, bool end_stream, + Filter& filter, FilterCallbacks& callbacks, + TimeSource& time_source) : coroutine_(coroutine), headers_(headers), end_stream_(end_stream), filter_(filter), callbacks_(callbacks), yield_callback_([this]() { if (state_ == State::Running) { @@ -224,7 +225,7 @@ Http::FilterDataStatus StreamHandleWrapper::onData(Buffer::Instance& data, bool if (state_ == State::WaitForBodyChunk) { ENVOY_LOG(trace, "resuming for next body chunk"); Filters::Common::Lua::LuaDeathRef wrapper( - Filters::Common::Lua::BufferWrapper::create(coroutine_.luaState(), data), true); + Filters::Common::Lua::BufferWrapper::create(coroutine_.luaState(), headers_, data), true); state_ = State::Running; coroutine_.resume(1, yield_callback_); } else if (state_ == State::WaitForBody && end_stream_) { @@ -458,9 +459,10 @@ int StreamHandleWrapper::luaBody(lua_State* state) { callbacks_.addData(body); } - body_wrapper_.reset(Filters::Common::Lua::BufferWrapper::create( - state, const_cast(*callbacks_.bufferedBody())), - true); + body_wrapper_.reset( + Filters::Common::Lua::BufferWrapper::create( + state, headers_, const_cast(*callbacks_.bufferedBody())), + true); } return 1; } @@ -720,11 +722,10 @@ void Filter::onDestroy() { } } -Http::FilterHeadersStatus Filter::doHeaders(StreamHandleRef& handle, - Filters::Common::Lua::CoroutinePtr& coroutine, - FilterCallbacks& callbacks, int function_ref, - PerLuaCodeSetup* setup, Http::HeaderMap& headers, - bool end_stream) { +Http::FilterHeadersStatus +Filter::doHeaders(StreamHandleRef& handle, Filters::Common::Lua::CoroutinePtr& coroutine, + FilterCallbacks& callbacks, int function_ref, PerLuaCodeSetup* setup, + Http::RequestOrResponseHeaderMap& headers, bool end_stream) { if (function_ref == LUA_REFNIL) { return Http::FilterHeadersStatus::Continue; } diff --git a/source/extensions/filters/http/lua/lua_filter.h b/source/extensions/filters/http/lua/lua_filter.h index a558177bd4418..c67e4f74560e3 100644 --- a/source/extensions/filters/http/lua/lua_filter.h +++ b/source/extensions/filters/http/lua/lua_filter.h @@ -133,9 +133,9 @@ class StreamHandleWrapper : public Filters::Common::Lua::BaseLuaObject { Http::FilterHeadersStatus doHeaders(StreamHandleRef& handle, Filters::Common::Lua::CoroutinePtr& coroutine, FilterCallbacks& callbacks, int function_ref, - PerLuaCodeSetup* setup, Http::HeaderMap& headers, - bool end_stream); + PerLuaCodeSetup* setup, + Http::RequestOrResponseHeaderMap& headers, bool end_stream); Http::FilterDataStatus doData(StreamHandleRef& handle, Buffer::Instance& data, bool end_stream); Http::FilterTrailersStatus doTrailers(StreamHandleRef& handle, Http::HeaderMap& trailers); diff --git a/test/extensions/filters/common/lua/wrappers_test.cc b/test/extensions/filters/common/lua/wrappers_test.cc index 565843d7e14c0..dc7f9e5ff2312 100644 --- a/test/extensions/filters/common/lua/wrappers_test.cc +++ b/test/extensions/filters/common/lua/wrappers_test.cc @@ -82,7 +82,8 @@ TEST_F(LuaBufferWrapperTest, Methods) { setup(SCRIPT); Buffer::OwnedImpl data("hello world"); - BufferWrapper::create(coroutine_->luaState(), data); + Http::TestRequestHeaderMapImpl headers; + BufferWrapper::create(coroutine_->luaState(), headers, data); EXPECT_CALL(printer_, testPrint("11")); EXPECT_CALL(printer_, testPrint("he")); EXPECT_CALL(printer_, testPrint("world")); @@ -101,7 +102,8 @@ TEST_F(LuaBufferWrapperTest, GetBytesInvalidParams) { setup(SCRIPT); Buffer::OwnedImpl data("hello world"); - BufferWrapper::create(coroutine_->luaState(), data); + Http::TestRequestHeaderMapImpl headers; + BufferWrapper::create(coroutine_->luaState(), headers, data); EXPECT_THROW_WITH_MESSAGE( start("callMe"), LuaException, "[string \"...\"]:3: index/length must be >= 0 and (index + length) must be <= buffer size"); diff --git a/test/extensions/filters/http/lua/lua_integration_test.cc b/test/extensions/filters/http/lua/lua_integration_test.cc index 14734bb5389fd..cc41bb5f353bc 100644 --- a/test/extensions/filters/http/lua/lua_integration_test.cc +++ b/test/extensions/filters/http/lua/lua_integration_test.cc @@ -1163,5 +1163,21 @@ name: lua testRewriteResponse(FILTER_AND_CODE); } +TEST_P(LuaIntegrationTest, RewriteResponseBufferWithoutHeaderReplaceContentLength) { + const std::string FILTER_AND_CODE = + R"EOF( +name: lua +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inline_code: | + function envoy_on_response(response_handle) + local content_length = response_handle:body():setBytes("ok") + response_handle:logTrace(content_length) + end +)EOF"; + + testRewriteResponse(FILTER_AND_CODE); +} + } // namespace } // namespace Envoy From 300137061360af3a4221d5d48f0d8bab2f3ad7f9 Mon Sep 17 00:00:00 2001 From: Tim Bart Date: Tue, 30 Nov 2021 06:45:39 -0800 Subject: [PATCH 31/51] ext-authz: fix missing UAEX flag on Denied CheckResponse (#18965) * ext-authz: fix missing UAEX flag on Denied CheckResponse This fixes a bug in which the UAEX flag is not set prior to calling `callback->sendLocalReply(...)`. Signed-off-by: Tim Bart --- docs/root/version_history/current.rst | 1 + source/extensions/filters/http/ext_authz/ext_authz.cc | 5 +++-- .../filters/http/ext_authz/ext_authz_test.cc | 10 ++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 1e4b8890da667..126a79c1bb971 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -26,6 +26,7 @@ Bug Fixes --------- *Changes expected to improve the state of the world and are unlikely to have negative effects* +* ext_authz: fix the ext_authz http filter to correctly set response flags to ``UAEX`` when a connection is denied. * ext_authz: fix the ext_authz network filter to correctly set response flag and code details to ``UAEX`` when a connection is denied. * listener: fixed the crash when updating listeners that do not bind to port. * tcp: fixed a bug where upstream circuit breakers applied HTTP per-request bounds to TCP connections. diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index 976f90786755e..9dbb86c015630 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -346,6 +346,9 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { config_->httpContext().codeStats().chargeResponseStat(info, false); } + // setResponseFlag must be called before sendLocalReply + decoder_callbacks_->streamInfo().setResponseFlag( + StreamInfo::ResponseFlag::UnauthorizedExternalService); decoder_callbacks_->sendLocalReply( response->status_code, response->body, [&headers = response->headers_to_set, @@ -365,8 +368,6 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { } }, absl::nullopt, Filters::Common::ExtAuthz::ResponseCodeDetails::get().AuthzDenied); - decoder_callbacks_->streamInfo().setResponseFlag( - StreamInfo::ResponseFlag::UnauthorizedExternalService); break; } diff --git a/test/extensions/filters/http/ext_authz/ext_authz_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_test.cc index 8f3033bd225a9..20ecd4331a11a 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_test.cc @@ -1916,14 +1916,15 @@ TEST_P(HttpFilterTestParam, DeniedResponseWith401) { Invoke([&](Filters::Common::ExtAuthz::RequestCallbacks& callbacks, const envoy::service::auth::v3::CheckRequest&, Tracing::Span&, const StreamInfo::StreamInfo&) -> void { request_callbacks_ = &callbacks; })); + + EXPECT_CALL(filter_callbacks_.stream_info_, + setResponseFlag(Envoy::StreamInfo::ResponseFlag::UnauthorizedExternalService)); EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, filter_->decodeHeaders(request_headers_, false)); Http::TestResponseHeaderMapImpl response_headers{{":status", "401"}}; EXPECT_CALL(filter_callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), true)); EXPECT_CALL(filter_callbacks_, continueDecoding()).Times(0); - EXPECT_CALL(filter_callbacks_.stream_info_, - setResponseFlag(Envoy::StreamInfo::ResponseFlag::UnauthorizedExternalService)); Filters::Common::ExtAuthz::Response response{}; response.status = Filters::Common::ExtAuthz::CheckStatus::Denied; @@ -1948,14 +1949,15 @@ TEST_P(HttpFilterTestParam, DeniedResponseWith403) { Invoke([&](Filters::Common::ExtAuthz::RequestCallbacks& callbacks, const envoy::service::auth::v3::CheckRequest&, Tracing::Span&, const StreamInfo::StreamInfo&) -> void { request_callbacks_ = &callbacks; })); + + EXPECT_CALL(filter_callbacks_.stream_info_, + setResponseFlag(Envoy::StreamInfo::ResponseFlag::UnauthorizedExternalService)); EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, filter_->decodeHeaders(request_headers_, false)); Http::TestResponseHeaderMapImpl response_headers{{":status", "403"}}; EXPECT_CALL(filter_callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), true)); EXPECT_CALL(filter_callbacks_, continueDecoding()).Times(0); - EXPECT_CALL(filter_callbacks_.stream_info_, - setResponseFlag(Envoy::StreamInfo::ResponseFlag::UnauthorizedExternalService)); Filters::Common::ExtAuthz::Response response{}; response.status = Filters::Common::ExtAuthz::CheckStatus::Denied; From c07bfeeabfff00ac93156ef57127dd8d17eae974 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Nov 2021 09:51:24 -0500 Subject: [PATCH 32/51] build(deps): bump sphinx from 4.3.0 to 4.3.1 in /tools/base (#19122) Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 4.3.0 to 4.3.1. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/4.x/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v4.3.0...v4.3.1) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/base/requirements.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt index 1cb42f984abb5..716b6135722c9 100644 --- a/tools/base/requirements.txt +++ b/tools/base/requirements.txt @@ -657,9 +657,9 @@ snowballstemmer==2.1.0 \ --hash=sha256:b51b447bea85f9968c13b650126a888aabd4cb4463fca868ec596826325dedc2 \ --hash=sha256:e997baa4f2e9139951b6f4c631bad912dfd3c792467e2f03d7239464af90e914 # via sphinx -sphinx==4.3.0 \ - --hash=sha256:6d051ab6e0d06cba786c4656b0fe67ba259fe058410f49e95bee6e49c4052cbf \ - --hash=sha256:7e2b30da5f39170efcd95c6270f07669d623c276521fee27ad6c380f49d2bf5b +sphinx==4.3.1 \ + --hash=sha256:048dac56039a5713f47a554589dc98a442b39226a2b9ed7f82797fcb2fe9253f \ + --hash=sha256:32a5b3e9a1b176cc25ed048557d4d3d01af635e6b76c5bc7a43b0a34447fbd45 # via # -r requirements.in # envoy.docs.sphinx-runner @@ -744,7 +744,6 @@ typing-extensions==3.10.0.2 \ # via # aiodocker # aiohttp - # gitpython uritemplate==3.0.1 \ --hash=sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f \ --hash=sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae From 55a97dd803555bf844b312071109bfef522e8b3c Mon Sep 17 00:00:00 2001 From: pradeepcrao <84025829+pradeepcrao@users.noreply.github.com> Date: Tue, 30 Nov 2021 15:13:40 +0000 Subject: [PATCH 33/51] Stats: Filter stats to be flushed to sinks (#18805) Provide the ability to filter stats to be flushed to sinks to reduce CPU usage for the periodic stats aggregation process. Commit Message: Additional Description: Envoy stats are periodically flushed to stat sinks (default cadence of 5s) on the main thread. The number of stats scales linearly with the number of clusters, as approximately 100 stats are replicated for each cluster. For high counts of clusters (order of 10k), the flushing of stats dominates CPU usage on the main thread. Being tied up in stats flushing can prevent the main thread from processing xDS updates in a timely manner, or even starve worker threads of CPU if the CPU is overcommitted. Usually, the number of stats of interest can be an order of magnitude lower than the number of stats. There is a mechanism to reject unwanted stats, but doing so will also make them unavailable for viewing in the admin console, which could hinder debuggability. Further, Envoy actually needs some of its stats to run (see for eg. #14610) which is currently an open bug. See the design doc below for more details: https://docs.google.com/document/d/1lzMvRlU5xY0yezpqA75N6kU747GY7I_WeGpBXiPaP5M/edit#heading=h.xgjl2srtytjt Risk Level: Low Testing: Added tests Docs Changes: NA Release Notes: NA Platform Specific Features: NA See below benchmark results from //test/server:server_stats_flush_benchmark ``` ---------------------------------------------------------------------------------- Benchmark Time CPU Iterations ---------------------------------------------------------------------------------- bmFlushToSinks/10 0.003 ms 0.003 ms 247626 bmFlushToSinks/100 0.019 ms 0.019 ms 36474 bmFlushToSinks/1000 0.193 ms 0.193 ms 3622 bmFlushToSinks/10000 2.25 ms 2.25 ms 299 bmFlushToSinks/100000 61.8 ms 61.8 ms 10 bmFlushToSinks/1000000 1212 ms 1212 ms 1 bmFlushToSinksWithPredicatesSet/10 0.001 ms 0.001 ms 496056 bmFlushToSinksWithPredicatesSet/100 0.007 ms 0.007 ms 104775 bmFlushToSinksWithPredicatesSet/1000 0.067 ms 0.067 ms 10411 bmFlushToSinksWithPredicatesSet/10000 0.704 ms 0.704 ms 938 bmFlushToSinksWithPredicatesSet/100000 28.0 ms 28.0 ms 25 bmFlushToSinksWithPredicatesSet/1000000 484 ms 484 ms 2 ``` Signed-off-by: Pradeep Rao --- envoy/server/instance.h | 12 ++ envoy/stats/allocator.h | 36 ++-- envoy/stats/sink.h | 23 +++ envoy/stats/stats.h | 10 ++ envoy/stats/store.h | 40 +++-- source/common/stats/allocator_impl.cc | 94 +++++++++- source/common/stats/allocator_impl.h | 24 ++- source/common/stats/isolated_store_impl.h | 28 ++- source/common/stats/thread_local_store.cc | 34 ++-- source/common/stats/thread_local_store.h | 18 +- source/server/config_validation/server.h | 1 + source/server/hot_restarting_parent.cc | 24 +-- source/server/server.cc | 7 +- source/server/server.h | 4 + test/common/stats/allocator_impl_test.cc | 166 +++++++++++++++++- test/common/stats/thread_local_store_test.cc | 5 +- test/integration/server.h | 26 ++- test/mocks/server/instance.h | 2 + test/mocks/stats/mocks.cc | 3 + test/mocks/stats/mocks.h | 19 +- test/server/config_validation/server_test.cc | 2 + .../server_stats_flush_benchmark_test.cc | 42 ++++- tools/spelling/spelling_dictionary.txt | 1 + 23 files changed, 527 insertions(+), 94 deletions(-) diff --git a/envoy/server/instance.h b/envoy/server/instance.h index 880aa4889b376..f779abb42a173 100644 --- a/envoy/server/instance.h +++ b/envoy/server/instance.h @@ -2,6 +2,7 @@ #include #include +#include #include #include "envoy/access_log/access_log.h" @@ -31,6 +32,11 @@ #include "envoy/upstream/cluster_manager.h" namespace Envoy { + +namespace Stats { +class SinkPredicates; +} + namespace Server { /** @@ -269,6 +275,12 @@ class Instance { * TODO(mattklein123): This can be removed when version 1.20.0 is no longer supported. */ virtual bool enableReusePortDefault() PURE; + + /** + * Set predicates for filtering counters, gauges and text readouts to be flushed to sinks. + */ + virtual void + setSinkPredicates(std::unique_ptr&& sink_predicates) PURE; }; } // namespace Server diff --git a/envoy/stats/allocator.h b/envoy/stats/allocator.h index 2924ebf0ab303..223b1cab47068 100644 --- a/envoy/stats/allocator.h +++ b/envoy/stats/allocator.h @@ -18,6 +18,9 @@ namespace Envoy { namespace Stats { +class Sink; +class SinkPredicates; + /** * Abstract interface for allocating statistics. Implementations can * be created utilizing a single fixed-size block suitable for @@ -70,19 +73,32 @@ class Allocator { virtual void markTextReadoutForDeletion(const TextReadoutSharedPtr& text_readout) PURE; /** - * Iterate over all stats that need to be added to a sink. Note, that implementations can + * Iterate over all stats. Note, that implementations can potentially hold on to a mutex that + * will deadlock if the passed in functors try to create or delete a stat. + * @param f_size functor that is provided the current number of all stats. Note that this is + * called only once, prior to any calls to f_stat. + * @param f_stat functor that is provided one stat at a time from the stats container. + */ + virtual void forEachCounter(SizeFn f_size, StatFn f_stat) const PURE; + virtual void forEachGauge(SizeFn f_size, StatFn f_stat) const PURE; + virtual void forEachTextReadout(SizeFn f_size, StatFn f_stat) const PURE; + + /** + * Iterate over all stats that need to be flushed to sinks. Note, that implementations can * potentially hold on to a mutex that will deadlock if the passed in functors try to create * or delete a stat. - * @param f_size functor that is provided the number of all stats in the sink. Note this is - * called only once, prior to any calls to f_stat. - * @param f_stat functor that is provided one stat in the sink at a time. + * @param f_size functor that is provided the number of all stats that will be flushed to sinks. + * Note that this is called only once, prior to any calls to f_stat. + * @param f_stat functor that is provided one stat that will be flushed to sinks, at a time. + */ + virtual void forEachSinkedCounter(SizeFn f_size, StatFn f_stat) const PURE; + virtual void forEachSinkedGauge(SizeFn f_size, StatFn f_stat) const PURE; + virtual void forEachSinkedTextReadout(SizeFn f_size, StatFn f_stat) const PURE; + + /** + * Set the predicates to filter counters, gauges and text readouts for sink. */ - virtual void forEachCounter(std::function f_size, - std::function f_stat) const PURE; - virtual void forEachGauge(std::function f_size, - std::function f_stat) const PURE; - virtual void forEachTextReadout(std::function f_size, - std::function f_stat) const PURE; + virtual void setSinkPredicates(std::unique_ptr&& sink_predicates) PURE; // TODO(jmarantz): create a parallel mechanism to instantiate histograms. At // the moment, histograms don't fit the same pattern of counters and gauges diff --git a/envoy/stats/sink.h b/envoy/stats/sink.h index ff0e607ffaa8c..a0d914416bd9d 100644 --- a/envoy/stats/sink.h +++ b/envoy/stats/sink.h @@ -48,6 +48,29 @@ class MetricSnapshot { virtual SystemTime snapshotTime() const PURE; }; +/** + * A class to define predicates to filter counters, gauges and text readouts for flushing to sinks. + */ +class SinkPredicates { +public: + virtual ~SinkPredicates() = default; + + /** + * @return true if @param counter needs to be flushed to sinks. + */ + virtual bool includeCounter(const Counter& counter) PURE; + + /** + * @return true if @param gague needs to be flushed to sinks. + */ + virtual bool includeGauge(const Gauge& gauge) PURE; + + /** + * @return true if @param text_readout needs to be flushed to sinks. + */ + virtual bool includeTextReadout(const TextReadout& text_readout) PURE; +}; + /** * A sink for stats. Each sink is responsible for writing stats to a backing store. */ diff --git a/envoy/stats/stats.h b/envoy/stats/stats.h index 4a9688b3fd42a..9fba64c34ccfa 100644 --- a/envoy/stats/stats.h +++ b/envoy/stats/stats.h @@ -190,5 +190,15 @@ class TextReadout : public virtual Metric { using TextReadoutSharedPtr = RefcountPtr; +/** + * Callback invoked to provide size of stats container. + */ +using SizeFn = std::function; + +/** + * Callback invoked for each stat during iteration. + */ +template using StatFn = std::function; + } // namespace Stats } // namespace Envoy diff --git a/envoy/stats/store.h b/envoy/stats/store.h index 3d456bbe7bec9..a54a170365787 100644 --- a/envoy/stats/store.h +++ b/envoy/stats/store.h @@ -24,6 +24,7 @@ class Instance; namespace Stats { class Sink; +class SinkPredicates; /** * A store for all known counters, gauges, and timers. @@ -51,20 +52,27 @@ class Store : public Scope { virtual std::vector histograms() const PURE; /** - * Iterate over all stats that need to be added to a sink. Note, that implementations can - * potentially hold on to a mutex that will deadlock if the passed in functors try to create - * or delete a stat. - * @param f_size functor that is provided the number of all stats in the sink. - * @param f_stat functor that is provided one stat in the sink at a time. + * Iterate over all stats. Note, that implementations can potentially hold on to a mutex that + * will deadlock if the passed in functors try to create or delete a stat. + * @param f_size functor that is provided the current number of all stats. Note that this is + * called only once, prior to any calls to f_stat. + * @param f_stat functor that is provided one stat at a time from the stats container. */ - virtual void forEachCounter(std::function f_size, - std::function f_stat) const PURE; - - virtual void forEachGauge(std::function f_size, - std::function f_stat) const PURE; + virtual void forEachCounter(SizeFn f_size, StatFn f_stat) const PURE; + virtual void forEachGauge(SizeFn f_size, StatFn f_stat) const PURE; + virtual void forEachTextReadout(SizeFn f_size, StatFn f_stat) const PURE; - virtual void forEachTextReadout(std::function f_size, - std::function f_stat) const PURE; + /** + * Iterate over all stats that need to be flushed to sinks. Note, that implementations can + * potentially hold on to a mutex that will deadlock if the passed in functors try to create + * or delete a stat. + * @param f_size functor that is provided the number of all stats that will be flushed to sinks. + * Note that this is called only once, prior to any calls to f_stat. + * @param f_stat functor that is provided one stat that will be flushed to sinks, at a time. + */ + virtual void forEachSinkedCounter(SizeFn f_size, StatFn f_stat) const PURE; + virtual void forEachSinkedGauge(SizeFn f_size, StatFn f_stat) const PURE; + virtual void forEachSinkedTextReadout(SizeFn f_size, StatFn f_stat) const PURE; }; using StorePtr = std::unique_ptr; @@ -123,6 +131,14 @@ class StoreRoot : public Store { * method would be asserted. */ virtual void mergeHistograms(PostMergeCb merge_complete_cb) PURE; + + /** + * Set predicates for filtering counters, gauges and text readouts to be flushed to sinks. + * Note that if the sink predicates object is set, we do not send non-sink stats over to the + * child process during hot restart. This will result in the admin stats console being wrong + * during hot restart. + */ + virtual void setSinkPredicates(std::unique_ptr&& sink_predicates) PURE; }; using StoreRootPtr = std::unique_ptr; diff --git a/source/common/stats/allocator_impl.cc b/source/common/stats/allocator_impl.cc index 9e8a37705e4d2..d0103b1065c7f 100644 --- a/source/common/stats/allocator_impl.cc +++ b/source/common/stats/allocator_impl.cc @@ -3,6 +3,7 @@ #include #include +#include "envoy/stats/sink.h" #include "envoy/stats/stats.h" #include "envoy/stats/symbol_table.h" @@ -144,6 +145,7 @@ class CounterImpl : public StatsSharedImpl { void removeFromSetLockHeld() ABSL_EXCLUSIVE_LOCKS_REQUIRED(alloc_.mutex_) override { const size_t count = alloc_.counters_.erase(statName()); ASSERT(count == 1); + alloc_.sinked_counters_.erase(this); } // Stats::Counter @@ -188,6 +190,7 @@ class GaugeImpl : public StatsSharedImpl { void removeFromSetLockHeld() override ABSL_EXCLUSIVE_LOCKS_REQUIRED(alloc_.mutex_) { const size_t count = alloc_.gauges_.erase(statName()); ASSERT(count == 1); + alloc_.sinked_gauges_.erase(this); } // Stats::Gauge @@ -260,6 +263,7 @@ class TextReadoutImpl : public StatsSharedImpl { void removeFromSetLockHeld() ABSL_EXCLUSIVE_LOCKS_REQUIRED(alloc_.mutex_) override { const size_t count = alloc_.text_readouts_.erase(statName()); ASSERT(count == 1); + alloc_.sinked_text_readouts_.erase(this); } // Stats::TextReadout @@ -289,6 +293,11 @@ CounterSharedPtr AllocatorImpl::makeCounter(StatName name, StatName tag_extracte } auto counter = CounterSharedPtr(makeCounterInternal(name, tag_extracted_name, stat_name_tags)); counters_.insert(counter.get()); + // Add counter to sinked_counters_ if it matches the sink predicate. + if (sink_predicates_ != nullptr && sink_predicates_->includeCounter(*counter)) { + auto val = sinked_counters_.insert(counter.get()); + ASSERT(val.second); + } return counter; } @@ -305,6 +314,11 @@ GaugeSharedPtr AllocatorImpl::makeGauge(StatName name, StatName tag_extracted_na auto gauge = GaugeSharedPtr(new GaugeImpl(name, *this, tag_extracted_name, stat_name_tags, import_mode)); gauges_.insert(gauge.get()); + // Add gauge to sinked_gauges_ if it matches the sink predicate. + if (sink_predicates_ != nullptr && sink_predicates_->includeGauge(*gauge)) { + auto val = sinked_gauges_.insert(gauge.get()); + ASSERT(val.second); + } return gauge; } @@ -320,6 +334,11 @@ TextReadoutSharedPtr AllocatorImpl::makeTextReadout(StatName name, StatName tag_ auto text_readout = TextReadoutSharedPtr(new TextReadoutImpl(name, *this, tag_extracted_name, stat_name_tags)); text_readouts_.insert(text_readout.get()); + // Add text_readout to sinked_text_readouts_ if it matches the sink predicate. + if (sink_predicates_ != nullptr && sink_predicates_->includeTextReadout(*text_readout)) { + auto val = sinked_text_readouts_.insert(text_readout.get()); + ASSERT(val.second); + } return text_readout; } @@ -336,8 +355,7 @@ Counter* AllocatorImpl::makeCounterInternal(StatName name, StatName tag_extracte return new CounterImpl(name, *this, tag_extracted_name, stat_name_tags); } -void AllocatorImpl::forEachCounter(std::function f_size, - std::function f_stat) const { +void AllocatorImpl::forEachCounter(SizeFn f_size, StatFn f_stat) const { Thread::LockGuard lock(mutex_); if (f_size != nullptr) { f_size(counters_.size()); @@ -347,8 +365,7 @@ void AllocatorImpl::forEachCounter(std::function f_size, } } -void AllocatorImpl::forEachGauge(std::function f_size, - std::function f_stat) const { +void AllocatorImpl::forEachGauge(SizeFn f_size, StatFn f_stat) const { Thread::LockGuard lock(mutex_); if (f_size != nullptr) { f_size(gauges_.size()); @@ -358,8 +375,7 @@ void AllocatorImpl::forEachGauge(std::function f_size, } } -void AllocatorImpl::forEachTextReadout(std::function f_size, - std::function f_stat) const { +void AllocatorImpl::forEachTextReadout(SizeFn f_size, StatFn f_stat) const { Thread::LockGuard lock(mutex_); if (f_size != nullptr) { f_size(text_readouts_.size()); @@ -369,6 +385,69 @@ void AllocatorImpl::forEachTextReadout(std::function f_size, } } +void AllocatorImpl::forEachSinkedCounter(SizeFn f_size, StatFn f_stat) const { + if (sink_predicates_ != nullptr) { + Thread::LockGuard lock(mutex_); + f_size(sinked_counters_.size()); + for (auto counter : sinked_counters_) { + f_stat(*counter); + } + } else { + forEachCounter(f_size, f_stat); + } +} + +void AllocatorImpl::forEachSinkedGauge(SizeFn f_size, StatFn f_stat) const { + if (sink_predicates_ != nullptr) { + Thread::LockGuard lock(mutex_); + f_size(sinked_gauges_.size()); + for (auto gauge : sinked_gauges_) { + f_stat(*gauge); + } + } else { + forEachGauge(f_size, f_stat); + } +} + +void AllocatorImpl::forEachSinkedTextReadout(SizeFn f_size, StatFn f_stat) const { + if (sink_predicates_ != nullptr) { + Thread::LockGuard lock(mutex_); + f_size(sinked_text_readouts_.size()); + for (auto text_readout : sinked_text_readouts_) { + f_stat(*text_readout); + } + } else { + forEachTextReadout(f_size, f_stat); + } +} + +void AllocatorImpl::setSinkPredicates(std::unique_ptr&& sink_predicates) { + Thread::LockGuard lock(mutex_); + ASSERT(sink_predicates_ == nullptr); + sink_predicates_ = std::move(sink_predicates); + sinked_counters_.clear(); + sinked_gauges_.clear(); + sinked_text_readouts_.clear(); + // Add counters to the set of sinked counters. + for (auto& counter : counters_) { + if (sink_predicates_->includeCounter(*counter)) { + sinked_counters_.emplace(counter); + } + } + // Add gauges to the set of sinked gauges. + for (auto& gauge : gauges_) { + if (sink_predicates_->includeGauge(*gauge)) { + sinked_gauges_.insert(gauge); + } + } + // Add text_readouts to the set of sinked text readouts. + for (auto& text_readout : text_readouts_) { + if (sink_predicates_->includeTextReadout(*text_readout)) { + sinked_text_readouts_.insert(text_readout); + } + } +} + void AllocatorImpl::markCounterForDeletion(const CounterSharedPtr& counter) { Thread::LockGuard lock(mutex_); auto iter = counters_.find(counter->statName()); @@ -380,6 +459,7 @@ void AllocatorImpl::markCounterForDeletion(const CounterSharedPtr& counter) { // Duplicates are ASSERTed in ~AllocatorImpl. deleted_counters_.emplace_back(*iter); counters_.erase(iter); + sinked_counters_.erase(counter.get()); } void AllocatorImpl::markGaugeForDeletion(const GaugeSharedPtr& gauge) { @@ -393,6 +473,7 @@ void AllocatorImpl::markGaugeForDeletion(const GaugeSharedPtr& gauge) { // Duplicates are ASSERTed in ~AllocatorImpl. deleted_gauges_.emplace_back(*iter); gauges_.erase(iter); + sinked_gauges_.erase(gauge.get()); } void AllocatorImpl::markTextReadoutForDeletion(const TextReadoutSharedPtr& text_readout) { @@ -406,6 +487,7 @@ void AllocatorImpl::markTextReadoutForDeletion(const TextReadoutSharedPtr& text_ // Duplicates are ASSERTed in ~AllocatorImpl. deleted_text_readouts_.emplace_back(*iter); text_readouts_.erase(iter); + sinked_text_readouts_.erase(text_readout.get()); } } // namespace Stats diff --git a/source/common/stats/allocator_impl.h b/source/common/stats/allocator_impl.h index 806e7dc8612fd..86a1e1aa664cf 100644 --- a/source/common/stats/allocator_impl.h +++ b/source/common/stats/allocator_impl.h @@ -2,7 +2,9 @@ #include +#include "envoy/common/optref.h" #include "envoy/stats/allocator.h" +#include "envoy/stats/sink.h" #include "envoy/stats/stats.h" #include "envoy/stats/symbol_table.h" @@ -33,15 +35,17 @@ class AllocatorImpl : public Allocator { SymbolTable& symbolTable() override { return symbol_table_; } const SymbolTable& constSymbolTable() const override { return symbol_table_; } - void forEachCounter(std::function, - std::function) const override; + void forEachCounter(SizeFn, StatFn) const override; - void forEachGauge(std::function, - std::function) const override; + void forEachGauge(SizeFn, StatFn) const override; - void forEachTextReadout(std::function, - std::function) const override; + void forEachTextReadout(SizeFn, StatFn) const override; + void forEachSinkedCounter(SizeFn f_size, StatFn f_stat) const override; + void forEachSinkedGauge(SizeFn f_size, StatFn f_stat) const override; + void forEachSinkedTextReadout(SizeFn f_size, StatFn f_stat) const override; + + void setSinkPredicates(std::unique_ptr&& sink_predicates) override; #ifndef ENVOY_CONFIG_COVERAGE void debugPrint(); #endif @@ -93,6 +97,14 @@ class AllocatorImpl : public Allocator { std::vector deleted_gauges_ ABSL_GUARDED_BY(mutex_); std::vector deleted_text_readouts_ ABSL_GUARDED_BY(mutex_); + template using StatPointerSet = absl::flat_hash_set; + // Stat pointers that participate in the flush to sink process. + StatPointerSet sinked_counters_ ABSL_GUARDED_BY(mutex_); + StatPointerSet sinked_gauges_ ABSL_GUARDED_BY(mutex_); + StatPointerSet sinked_text_readouts_ ABSL_GUARDED_BY(mutex_); + + // Predicates used to filter stats to be flushed. + std::unique_ptr sink_predicates_; SymbolTable& symbol_table_; Thread::ThreadSynchronizer sync_; diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index ebff944da7eff..cc44d8e811cac 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -101,9 +101,10 @@ template class IsolatedStatsCache { return true; } - void forEachStat(std::function f_size, - std::function f_stat) const { - f_size(stats_.size()); + void forEachStat(SizeFn f_size, std::function f_stat) const { + if (f_size != nullptr) { + f_size(stats_.size()); + } for (auto const& stat : stats_) { f_stat(*stat.second); } @@ -214,21 +215,30 @@ class IsolatedStoreImpl : public StoreImpl { return textReadoutFromStatName(storage.statName()); } - void forEachCounter(std::function f_size, - std::function f_stat) const override { + void forEachCounter(SizeFn f_size, StatFn f_stat) const override { counters_.forEachStat(f_size, f_stat); } - void forEachGauge(std::function f_size, - std::function f_stat) const override { + void forEachGauge(SizeFn f_size, StatFn f_stat) const override { gauges_.forEachStat(f_size, f_stat); } - void forEachTextReadout(std::function f_size, - std::function f_stat) const override { + void forEachTextReadout(SizeFn f_size, StatFn f_stat) const override { text_readouts_.forEachStat(f_size, f_stat); } + void forEachSinkedCounter(SizeFn f_size, StatFn f_stat) const override { + forEachCounter(f_size, f_stat); + } + + void forEachSinkedGauge(SizeFn f_size, StatFn f_stat) const override { + forEachGauge(f_size, f_stat); + } + + void forEachSinkedTextReadout(SizeFn f_size, StatFn f_stat) const override { + forEachTextReadout(f_size, f_stat); + } + private: IsolatedStoreImpl(std::unique_ptr&& symbol_table); diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index 97960182096c0..f0c32033ceccf 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -954,24 +954,38 @@ bool ParentHistogramImpl::usedLockHeld() const { return false; } -void ThreadLocalStoreImpl::forEachCounter(std::function f_size, - std::function f_stat) const { - Thread::LockGuard lock(lock_); +void ThreadLocalStoreImpl::forEachCounter(SizeFn f_size, StatFn f_stat) const { alloc_.forEachCounter(f_size, f_stat); } -void ThreadLocalStoreImpl::forEachGauge(std::function f_size, - std::function f_stat) const { - Thread::LockGuard lock(lock_); +void ThreadLocalStoreImpl::forEachGauge(SizeFn f_size, StatFn f_stat) const { alloc_.forEachGauge(f_size, f_stat); } -void ThreadLocalStoreImpl::forEachTextReadout( - std::function f_size, - std::function f_stat) const { - Thread::LockGuard lock(lock_); +void ThreadLocalStoreImpl::forEachTextReadout(SizeFn f_size, StatFn f_stat) const { alloc_.forEachTextReadout(f_size, f_stat); } +void ThreadLocalStoreImpl::forEachSinkedCounter(SizeFn f_size, StatFn f_stat) const { + alloc_.forEachSinkedCounter(f_size, f_stat); +} + +void ThreadLocalStoreImpl::forEachSinkedGauge(SizeFn f_size, StatFn f_stat) const { + alloc_.forEachSinkedGauge(f_size, f_stat); +} + +void ThreadLocalStoreImpl::forEachSinkedTextReadout(SizeFn f_size, + StatFn f_stat) const { + alloc_.forEachSinkedTextReadout(f_size, f_stat); +} + +void ThreadLocalStoreImpl::setSinkPredicates(std::unique_ptr&& sink_predicates) { + ASSERT(sink_predicates != nullptr); + if (sink_predicates != nullptr) { + sink_predicates_.emplace(*sink_predicates); + alloc_.setSinkPredicates(std::move(sink_predicates)); + } +} + } // namespace Stats } // namespace Envoy diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index 742e1fc3c04d6..a22a04d00f985 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -244,14 +244,9 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo std::vector textReadouts() const override; std::vector histograms() const override; - void forEachCounter(std::function f_size, - std::function f_stat) const override; - - void forEachGauge(std::function f_size, - std::function f_stat) const override; - - void forEachTextReadout(std::function f_size, - std::function f_stat) const override; + void forEachCounter(SizeFn f_size, StatFn f_stat) const override; + void forEachGauge(SizeFn f_size, StatFn f_stat) const override; + void forEachTextReadout(SizeFn f_size, StatFn f_stat) const override; // Stats::StoreRoot void addSink(Sink& sink) override { timer_sinks_.push_back(sink); } @@ -267,6 +262,12 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo Histogram& tlsHistogram(ParentHistogramImpl& parent, uint64_t id); + void forEachSinkedCounter(SizeFn f_size, StatFn f_stat) const override; + void forEachSinkedGauge(SizeFn f_size, StatFn f_stat) const override; + void forEachSinkedTextReadout(SizeFn f_size, StatFn f_stat) const override; + + void setSinkPredicates(std::unique_ptr&& sink_predicates) override; + /** * @return a thread synchronizer object used for controlling thread behavior in tests. */ @@ -500,6 +501,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo StatNameHashSet* tls_rejected_stats); TlsCache& tlsCache() { return **tls_cache_; } + OptRef sink_predicates_; Allocator& alloc_; Event::Dispatcher* main_thread_dispatcher_{}; using TlsCacheSlot = ThreadLocal::TypedSlotPtr; diff --git a/source/server/config_validation/server.h b/source/server/config_validation/server.h index a48faf8554238..6df0b4c67e725 100644 --- a/source/server/config_validation/server.h +++ b/source/server/config_validation/server.h @@ -125,6 +125,7 @@ class ValidationInstance final : Logger::Loggable, void setDefaultTracingConfig(const envoy::config::trace::v3::Tracing& tracing_config) override { http_context_.setDefaultTracingConfig(tracing_config); } + void setSinkPredicates(std::unique_ptr&&) override {} // Server::ListenerComponentFactory LdsApiPtr createLdsApi(const envoy::config::core::v3::ConfigSource& lds_config, diff --git a/source/server/hot_restarting_parent.cc b/source/server/hot_restarting_parent.cc index 14c012b70cbca..fab51c4fc7b53 100644 --- a/source/server/hot_restarting_parent.cc +++ b/source/server/hot_restarting_parent.cc @@ -128,26 +128,26 @@ HotRestartingParent::Internal::getListenSocketsForChild(const HotRestartMessage: // magnitude of memory usage that they are meant to avoid, since this map holds full-string // names. The problem can be solved by splitting the export up over many chunks. void HotRestartingParent::Internal::exportStatsToChild(HotRestartMessage::Reply::Stats* stats) { - for (const auto& gauge : server_->stats().gauges()) { - if (gauge->used()) { - const std::string name = gauge->name(); - (*stats->mutable_gauges())[name] = gauge->value(); - recordDynamics(stats, name, gauge->statName()); + server_->stats().forEachSinkedGauge(nullptr, [this, stats](Stats::Gauge& gauge) mutable { + if (gauge.used()) { + const std::string name = gauge.name(); + (*stats->mutable_gauges())[name] = gauge.value(); + recordDynamics(stats, name, gauge.statName()); } - } + }); - for (const auto& counter : server_->stats().counters()) { - if (counter->used()) { + server_->stats().forEachSinkedCounter(nullptr, [this, stats](Stats::Counter& counter) mutable { + if (counter.used()) { // The hot restart parent is expected to have stopped its normal stat exporting (and so // latching) by the time it begins exporting to the hot restart child. - uint64_t latched_value = counter->latch(); + uint64_t latched_value = counter.latch(); if (latched_value > 0) { - const std::string name = counter->name(); + const std::string name = counter.name(); (*stats->mutable_counter_deltas())[name] = latched_value; - recordDynamics(stats, name, counter->statName()); + recordDynamics(stats, name, counter.statName()); } } - } + }); stats->set_memory_allocated(Memory::Stats::totalCurrentlyAllocated()); stats->set_num_connections(server_->listenerManager().numConnections()); } diff --git a/source/server/server.cc b/source/server/server.cc index c914699197e6b..98973229f182a 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -20,6 +20,7 @@ #include "envoy/server/bootstrap_extension_config.h" #include "envoy/server/instance.h" #include "envoy/server/options.h" +#include "envoy/stats/histogram.h" #include "envoy/stats/stats.h" #include "envoy/upstream/cluster_manager.h" @@ -168,7 +169,7 @@ void InstanceImpl::failHealthcheck(bool fail) { } MetricSnapshotImpl::MetricSnapshotImpl(Stats::Store& store, TimeSource& time_source) { - store.forEachCounter( + store.forEachSinkedCounter( [this](std::size_t size) mutable { snapped_counters_.reserve(size); counters_.reserve(size); @@ -178,7 +179,7 @@ MetricSnapshotImpl::MetricSnapshotImpl(Stats::Store& store, TimeSource& time_sou counters_.push_back({counter.latch(), counter}); }); - store.forEachGauge( + store.forEachSinkedGauge( [this](std::size_t size) mutable { snapped_gauges_.reserve(size); gauges_.reserve(size); @@ -195,7 +196,7 @@ MetricSnapshotImpl::MetricSnapshotImpl(Stats::Store& store, TimeSource& time_sou histograms_.push_back(*histogram); } - store.forEachTextReadout( + store.forEachSinkedTextReadout( [this](std::size_t size) mutable { snapped_text_readouts_.reserve(size); text_readouts_.reserve(size); diff --git a/source/server/server.h b/source/server/server.h index 19ac1b5ce169a..5fa20d0a0cd01 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -293,6 +293,10 @@ class InstanceImpl final : Logger::Loggable, Quic::QuicStatNames& quicStatNames() { return quic_stat_names_; } + void setSinkPredicates(std::unique_ptr&& sink_predicates) override { + stats_store_.setSinkPredicates(std::move(sink_predicates)); + } + // ServerLifecycleNotifier ServerLifecycleNotifier::HandlePtr registerCallback(Stage stage, StageCallback callback) override; ServerLifecycleNotifier::HandlePtr diff --git a/test/common/stats/allocator_impl_test.cc b/test/common/stats/allocator_impl_test.cc index cc06acedaef1e..a4c0eb2d271b4 100644 --- a/test/common/stats/allocator_impl_test.cc +++ b/test/common/stats/allocator_impl_test.cc @@ -1,6 +1,9 @@ #include +#include #include +#include "envoy/stats/sink.h" + #include "source/common/stats/allocator_impl.h" #include "test/test_common/logging.h" @@ -41,6 +44,26 @@ class AllocatorImplTest : public testing::Test { bool are_stats_marked_for_deletion_ = false; }; +class TestSinkPredicates : public SinkPredicates { +public: + ~TestSinkPredicates() override = default; + StatNameHashSet& sinkedStatNames() { return sinked_stat_names_; } + + // SinkPredicates + bool includeCounter(const Counter& counter) override { + return sinked_stat_names_.find(counter.statName()) != sinked_stat_names_.end(); + } + bool includeGauge(const Gauge& gauge) override { + return sinked_stat_names_.find(gauge.statName()) != sinked_stat_names_.end(); + } + bool includeTextReadout(const TextReadout& text_readout) override { + return sinked_stat_names_.find(text_readout.statName()) != sinked_stat_names_.end(); + } + +private: + StatNameHashSet sinked_stat_names_; +}; + // Allocate 2 counters of the same name, and you'll get the same object. TEST_F(AllocatorImplTest, CountersWithSameName) { StatName counter_name = makeStat("counter.name"); @@ -309,7 +332,7 @@ TEST_F(AllocatorImplTest, ForEachWithNullSizeLambda) { } size_t num_iterations = 0; alloc_.forEachCounter(nullptr, [&num_iterations](Stats::Counter& counter) { - (void)counter; + UNREFERENCED_PARAMETER(counter); ++num_iterations; }); EXPECT_EQ(num_iterations, num_stats); @@ -321,7 +344,7 @@ TEST_F(AllocatorImplTest, ForEachWithNullSizeLambda) { } num_iterations = 0; alloc_.forEachGauge(nullptr, [&num_iterations](Stats::Gauge& gauge) { - (void)gauge; + UNREFERENCED_PARAMETER(gauge); ++num_iterations; }); EXPECT_EQ(num_iterations, num_stats); @@ -333,7 +356,7 @@ TEST_F(AllocatorImplTest, ForEachWithNullSizeLambda) { } num_iterations = 0; alloc_.forEachTextReadout(nullptr, [&num_iterations](Stats::TextReadout& text_readout) { - (void)text_readout; + UNREFERENCED_PARAMETER(text_readout); ++num_iterations; }); EXPECT_EQ(num_iterations, num_stats); @@ -410,6 +433,143 @@ TEST_F(AllocatorImplTest, AskForDeletedStat) { EXPECT_EQ(rejected_text_readout.value(), "deleted value"); } +TEST_F(AllocatorImplTest, ForEachSinkedCounter) { + std::unique_ptr moved_sink_predicates = + std::make_unique(); + TestSinkPredicates* sink_predicates = moved_sink_predicates.get(); + std::vector sinked_counters; + std::vector unsinked_counters; + + alloc_.setSinkPredicates(std::move(moved_sink_predicates)); + + const size_t num_stats = 11; + + for (size_t idx = 0; idx < num_stats; ++idx) { + auto stat_name = makeStat(absl::StrCat("counter.", idx)); + // sink every 3rd stat + if ((idx + 1) % 3 == 0) { + sink_predicates->sinkedStatNames().insert(stat_name); + sinked_counters.emplace_back(alloc_.makeCounter(stat_name, StatName(), {})); + } else { + unsinked_counters.emplace_back(alloc_.makeCounter(stat_name, StatName(), {})); + } + } + + EXPECT_EQ(sinked_counters.size(), 3); + EXPECT_EQ(unsinked_counters.size(), 8); + + size_t num_sinked_counters = 0; + size_t num_iterations = 0; + alloc_.forEachSinkedCounter( + [&num_sinked_counters](std::size_t size) { num_sinked_counters = size; }, + [&num_iterations, sink_predicates](Stats::Counter& counter) { + EXPECT_EQ(sink_predicates->sinkedStatNames().count(counter.statName()), 1); + ++num_iterations; + }); + EXPECT_EQ(num_sinked_counters, 3); + EXPECT_EQ(num_iterations, 3); + + // Erase all sinked stats. + sinked_counters.clear(); + num_iterations = 0; + alloc_.forEachSinkedCounter( + [&num_sinked_counters](std::size_t size) { num_sinked_counters = size; }, + [&num_iterations](Stats::Counter&) { ++num_iterations; }); + EXPECT_EQ(num_sinked_counters, 0); + EXPECT_EQ(num_iterations, 0); +} + +TEST_F(AllocatorImplTest, ForEachSinkedGauge) { + std::unique_ptr moved_sink_predicates = + std::make_unique(); + TestSinkPredicates* sink_predicates = moved_sink_predicates.get(); + std::vector sinked_gauges; + std::vector unsinked_gauges; + + alloc_.setSinkPredicates(std::move(moved_sink_predicates)); + const size_t num_stats = 11; + + for (size_t idx = 0; idx < num_stats; ++idx) { + auto stat_name = makeStat(absl::StrCat("gauge.", idx)); + // sink every 5th stat + if ((idx + 1) % 5 == 0) { + sink_predicates->sinkedStatNames().insert(stat_name); + sinked_gauges.emplace_back( + alloc_.makeGauge(stat_name, StatName(), {}, Gauge::ImportMode::Accumulate)); + } else { + unsinked_gauges.emplace_back( + alloc_.makeGauge(stat_name, StatName(), {}, Gauge::ImportMode::Accumulate)); + } + } + + EXPECT_EQ(sinked_gauges.size(), 2); + EXPECT_EQ(unsinked_gauges.size(), 9); + + size_t num_sinked_gauges = 0; + size_t num_iterations = 0; + alloc_.forEachSinkedGauge([&num_sinked_gauges](std::size_t size) { num_sinked_gauges = size; }, + [&num_iterations, sink_predicates](Stats::Gauge& gauge) { + EXPECT_EQ(sink_predicates->sinkedStatNames().count(gauge.statName()), + 1); + ++num_iterations; + }); + EXPECT_EQ(num_sinked_gauges, 2); + EXPECT_EQ(num_iterations, 2); + + // Erase all sinked stats. + sinked_gauges.clear(); + num_iterations = 0; + alloc_.forEachSinkedGauge([&num_sinked_gauges](std::size_t size) { num_sinked_gauges = size; }, + [&num_iterations](Stats::Gauge&) { ++num_iterations; }); + EXPECT_EQ(num_sinked_gauges, 0); + EXPECT_EQ(num_iterations, 0); +} + +TEST_F(AllocatorImplTest, ForEachSinkedTextReadout) { + std::unique_ptr moved_sink_predicates = + std::make_unique(); + TestSinkPredicates* sink_predicates = moved_sink_predicates.get(); + std::vector sinked_text_readouts; + std::vector unsinked_text_readouts; + + alloc_.setSinkPredicates(std::move(moved_sink_predicates)); + const size_t num_stats = 11; + + for (size_t idx = 0; idx < num_stats; ++idx) { + auto stat_name = makeStat(absl::StrCat("text_readout.", idx)); + // sink every 2nd stat + if ((idx + 1) % 2 == 0) { + sink_predicates->sinkedStatNames().insert(stat_name); + sinked_text_readouts.emplace_back(alloc_.makeTextReadout(stat_name, StatName(), {})); + } else { + unsinked_text_readouts.emplace_back(alloc_.makeTextReadout(stat_name, StatName(), {})); + } + } + + EXPECT_EQ(sinked_text_readouts.size(), 5); + EXPECT_EQ(unsinked_text_readouts.size(), 6); + + size_t num_sinked_text_readouts = 0; + size_t num_iterations = 0; + alloc_.forEachSinkedTextReadout( + [&num_sinked_text_readouts](std::size_t size) { num_sinked_text_readouts = size; }, + [&num_iterations, sink_predicates](Stats::TextReadout& text_readout) { + EXPECT_EQ(sink_predicates->sinkedStatNames().count(text_readout.statName()), 1); + ++num_iterations; + }); + EXPECT_EQ(num_sinked_text_readouts, 5); + EXPECT_EQ(num_iterations, 5); + + // Erase all sinked stats. + sinked_text_readouts.clear(); + num_iterations = 0; + alloc_.forEachSinkedTextReadout( + [&num_sinked_text_readouts](std::size_t size) { num_sinked_text_readouts = size; }, + [&num_iterations](Stats::TextReadout&) { ++num_iterations; }); + EXPECT_EQ(num_sinked_text_readouts, 0); + EXPECT_EQ(num_iterations, 0); +} + } // namespace } // namespace Stats } // namespace Envoy diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 0847d9605df29..a5a59f960c781 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -1,9 +1,11 @@ #include +#include #include #include #include "envoy/config/metrics/v3/stats.pb.h" #include "envoy/stats/histogram.h" +#include "envoy/stats/sink.h" #include "source/common/common/c_smart_ptr.h" #include "source/common/event/dispatcher_impl.h" @@ -1564,6 +1566,7 @@ TEST_F(HistogramTest, ParentHistogramBucketSummary) { "B3.6e+06(1,1)", parent_histogram->bucketSummary()); } + class ThreadLocalRealThreadsTestBase : public Thread::RealThreadsTestHelper, public ThreadLocalStoreNoMocksTestBase { protected: @@ -1801,7 +1804,7 @@ TEST_F(HistogramThreadTest, ScopeOverlap) { 2 * NumThreads, ") "))); // Now clear everything, and synchronize the system by calling mergeHistograms(). - // THere should be no more ParentHistograms or TlsHistograms. + // There should be no more ParentHistograms or TlsHistograms. scope2.reset(); histograms.clear(); mergeHistograms(); diff --git a/test/integration/server.h b/test/integration/server.h index 880b899d2f3a9..48e4c5eb08ae4 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -9,6 +9,7 @@ #include "envoy/config/listener/v3/listener.pb.h" #include "envoy/server/options.h" #include "envoy/server/process_context.h" +#include "envoy/stats/histogram.h" #include "envoy/stats/stats.h" #include "source/common/common/assert.h" @@ -281,21 +282,34 @@ class TestIsolatedStoreImpl : public StoreRoot { Thread::LockGuard lock(lock_); return store_.counterFromStatNameWithTags(name, tags); } - void forEachCounter(std::function f_size, - std::function f_stat) const override { + void forEachCounter(Stats::SizeFn f_size, StatFn f_stat) const override { Thread::LockGuard lock(lock_); store_.forEachCounter(f_size, f_stat); } - void forEachGauge(std::function f_size, - std::function f_stat) const override { + void forEachGauge(Stats::SizeFn f_size, StatFn f_stat) const override { Thread::LockGuard lock(lock_); store_.forEachGauge(f_size, f_stat); } - void forEachTextReadout(std::function f_size, - std::function f_stat) const override { + void forEachTextReadout(Stats::SizeFn f_size, StatFn f_stat) const override { Thread::LockGuard lock(lock_); store_.forEachTextReadout(f_size, f_stat); } + void forEachSinkedCounter(Stats::SizeFn f_size, StatFn f_stat) const override { + Thread::LockGuard lock(lock_); + store_.forEachSinkedCounter(f_size, f_stat); + } + void forEachSinkedGauge(Stats::SizeFn f_size, StatFn f_stat) const override { + Thread::LockGuard lock(lock_); + store_.forEachSinkedGauge(f_size, f_stat); + } + void forEachSinkedTextReadout(Stats::SizeFn f_size, StatFn f_stat) const override { + Thread::LockGuard lock(lock_); + store_.forEachSinkedTextReadout(f_size, f_stat); + } + void setSinkPredicates(std::unique_ptr&& sink_predicates) override { + UNREFERENCED_PARAMETER(sink_predicates); + } + Counter& counterFromString(const std::string& name) override { Thread::LockGuard lock(lock_); return store_.counterFromString(name); diff --git a/test/mocks/server/instance.h b/test/mocks/server/instance.h index de4f51099823c..c208614218726 100644 --- a/test/mocks/server/instance.h +++ b/test/mocks/server/instance.h @@ -87,6 +87,7 @@ class MockInstance : public Instance { MOCK_METHOD(Configuration::ServerFactoryContext&, serverFactoryContext, ()); MOCK_METHOD(Configuration::TransportSocketFactoryContext&, transportSocketFactoryContext, ()); MOCK_METHOD(bool, enableReusePortDefault, ()); + MOCK_METHOD(void, setSinkPredicates, (std::unique_ptr &&)); void setDefaultTracingConfig(const envoy::config::trace::v3::Tracing& tracing_config) override { http_context_.setDefaultTracingConfig(tracing_config); @@ -141,6 +142,7 @@ class MockStatsConfig : public virtual StatsConfig { MOCK_METHOD(const std::list&, sinks, (), (const)); MOCK_METHOD(std::chrono::milliseconds, flushInterval, (), (const)); MOCK_METHOD(bool, flushOnAdmin, (), (const)); + MOCK_METHOD(const Stats::SinkPredicates*, sinkPredicates, (), (const)); }; class MockServerFactoryContext : public virtual ServerFactoryContext { diff --git a/test/mocks/stats/mocks.cc b/test/mocks/stats/mocks.cc index 7eba3fc31cda2..6ed277eed2fd1 100644 --- a/test/mocks/stats/mocks.cc +++ b/test/mocks/stats/mocks.cc @@ -70,6 +70,9 @@ MockMetricSnapshot::~MockMetricSnapshot() = default; MockSink::MockSink() = default; MockSink::~MockSink() = default; +MockSinkPredicates::MockSinkPredicates() = default; +MockSinkPredicates::~MockSinkPredicates() = default; + MockStore::MockStore() { ON_CALL(*this, counter(_)).WillByDefault(ReturnRef(counter_)); ON_CALL(*this, gauge(_, _)).WillByDefault(ReturnRef(gauge_)); diff --git a/test/mocks/stats/mocks.h b/test/mocks/stats/mocks.h index 3043a2c35fbb4..fd3d245395510 100644 --- a/test/mocks/stats/mocks.h +++ b/test/mocks/stats/mocks.h @@ -264,6 +264,15 @@ class MockSink : public Sink { MOCK_METHOD(void, onHistogramComplete, (const Histogram& histogram, uint64_t value)); }; +class MockSinkPredicates : public SinkPredicates { +public: + MockSinkPredicates(); + ~MockSinkPredicates() override; + MOCK_METHOD(bool, includeCounter, (const Counter&)); + MOCK_METHOD(bool, includeGauge, (const Gauge&)); + MOCK_METHOD(bool, includeTextReadout, (const TextReadout&)); +}; + class MockStore : public TestUtil::TestStore { public: MockStore(); @@ -286,13 +295,9 @@ class MockStore : public TestUtil::TestStore { MOCK_METHOD(Histogram&, histogramFromString, (const std::string& name, Histogram::Unit unit)); MOCK_METHOD(TextReadout&, textReadout, (const std::string&)); MOCK_METHOD(std::vector, text_readouts, (), (const)); - MOCK_METHOD(void, forEachCounter, - (std::function, std::function), (const)); - MOCK_METHOD(void, forEachGauge, - (std::function, std::function), (const)); - MOCK_METHOD(void, forEachTextReadout, - (std::function, std::function), - (const)); + MOCK_METHOD(void, forEachCounter, (SizeFn, StatFn), (const)); + MOCK_METHOD(void, forEachGauge, (SizeFn, StatFn), (const)); + MOCK_METHOD(void, forEachTextReadout, (SizeFn, StatFn), (const)); MOCK_METHOD(CounterOptConstRef, findCounter, (StatName), (const)); MOCK_METHOD(GaugeOptConstRef, findGauge, (StatName), (const)); diff --git a/test/server/config_validation/server_test.cc b/test/server/config_validation/server_test.cc index 14611beda5a60..d65e7354fa651 100644 --- a/test/server/config_validation/server_test.cc +++ b/test/server/config_validation/server_test.cc @@ -1,3 +1,4 @@ +#include #include #include "envoy/server/filter_config.h" @@ -129,6 +130,7 @@ TEST_P(ValidationServerTest, NoopLifecycleNotifier) { server.registerCallback(ServerLifecycleNotifier::Stage::ShutdownExit, [] { FAIL(); }); server.registerCallback(ServerLifecycleNotifier::Stage::ShutdownExit, [](Event::PostCb) { FAIL(); }); + server.setSinkPredicates(std::make_unique>()); server.shutdown(); } diff --git a/test/server/server_stats_flush_benchmark_test.cc b/test/server/server_stats_flush_benchmark_test.cc index 98790c5c65d56..895967523c624 100644 --- a/test/server/server_stats_flush_benchmark_test.cc +++ b/test/server/server_stats_flush_benchmark_test.cc @@ -19,10 +19,28 @@ namespace Envoy { +class TestSinkPredicates : public Stats::SinkPredicates { +public: + bool includeCounter(const Stats::Counter&) override { return (++num_counters_) % 10 == 0; } + bool includeGauge(const Stats::Gauge&) override { return (++num_gauges_) % 10 == 0; } + bool includeTextReadout(const Stats::TextReadout&) override { + return (++num_text_readouts_) % 10 == 0; + } + +private: + size_t num_counters_ = 0; + size_t num_gauges_ = 0; + size_t num_text_readouts_ = 0; +}; + class StatsSinkFlushSpeedTest { public: - StatsSinkFlushSpeedTest(size_t const num_stats) + StatsSinkFlushSpeedTest(size_t const num_stats, bool set_sink_predicates = false) : pool_(symbol_table_), stats_allocator_(symbol_table_), stats_store_(stats_allocator_) { + if (set_sink_predicates) { + stats_store_.setSinkPredicates( + std::unique_ptr{std::make_unique()}); + } // Create counters for (uint64_t idx = 0; idx < num_stats; ++idx) { @@ -40,6 +58,12 @@ class StatsSinkFlushSpeedTest { auto stat_name = pool_.add(absl::StrCat("text_readout.", idx)); stats_store_.textReadoutFromStatName(stat_name).set(absl::StrCat("text_readout.", idx)); } + + // Create histograms + for (uint64_t idx = 0; idx < num_stats; ++idx) { + std::string stat_name(absl::StrCat("histogram.", idx)); + stats_store_.histogramFromString(stat_name, Stats::Histogram::Unit::Unspecified); + } } void test(::benchmark::State& state) { @@ -69,6 +93,22 @@ static void bmFlushToSinks(::benchmark::State& state) { StatsSinkFlushSpeedTest speed_test(state.range(0)); speed_test.test(state); } + +static void bmFlushToSinksWithPredicatesSet(::benchmark::State& state) { + // Skip expensive benchmarks for unit tests. + if (benchmark::skipExpensiveBenchmarks() && state.range(0) > 100) { + state.SkipWithError("Skipping expensive benchmark"); + return; + } + + StatsSinkFlushSpeedTest speed_test(state.range(0), true); + speed_test.test(state); +} + BENCHMARK(bmFlushToSinks)->Unit(::benchmark::kMillisecond)->RangeMultiplier(10)->Range(10, 1000000); +BENCHMARK(bmFlushToSinksWithPredicatesSet) + ->Unit(::benchmark::kMillisecond) + ->RangeMultiplier(10) + ->Range(10, 1000000); } // namespace Envoy diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 80ca75656c1e0..0e4e9aaf1e3e1 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -1109,6 +1109,7 @@ siginfo signalstack siloed sim +sinked sizeof smatch snapshotted From 76a70b40f57bd9a75b50d4783d28dec0e0aa29ae Mon Sep 17 00:00:00 2001 From: Max Kuznetsov Date: Tue, 30 Nov 2021 11:06:01 -0500 Subject: [PATCH 34/51] Extend StatefulHeaderFormatter to allow forwarding HTTP1 reason phrase (#18997) Signed-off-by: Max Kuznetsov --- .../preserve_case/v3/preserve_case.proto | 3 + docs/root/version_history/current.rst | 1 + envoy/http/header_formatter.h | 10 ++ source/common/http/http1/codec_impl.cc | 22 ++++- source/common/http/http1/codec_impl.h | 2 + .../common/http/http1/legacy_parser_impl.cc | 6 +- source/common/http/http1/parser.h | 8 ++ .../preserve_case/preserve_case_formatter.cc | 34 ++++++- .../preserve_case/preserve_case_formatter.h | 7 ++ .../header_formatters/preserve_case/BUILD | 14 +++ ...ormatter_reason_phrase_integration_test.cc | 93 +++++++++++++++++++ .../preserve_case_formatter_test.cc | 18 +++- 12 files changed, 210 insertions(+), 8 deletions(-) create mode 100644 test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_reason_phrase_integration_test.cc diff --git a/api/envoy/extensions/http/header_formatters/preserve_case/v3/preserve_case.proto b/api/envoy/extensions/http/header_formatters/preserve_case/v3/preserve_case.proto index 64bdd497ecab0..5bd3691416374 100644 --- a/api/envoy/extensions/http/header_formatters/preserve_case/v3/preserve_case.proto +++ b/api/envoy/extensions/http/header_formatters/preserve_case/v3/preserve_case.proto @@ -16,4 +16,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // See the :ref:`header casing ` configuration guide for more // information. message PreserveCaseFormatterConfig { + // Allows forwarding reason phrase text. + // This is off by default, and a standard reason phrase is used for a corresponding HTTP response code. + bool forward_reason_phrase = 1; } diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 126a79c1bb971..ddaed36ae2dcd 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -70,6 +70,7 @@ New Features * http: added support for %REQUESTED_SERVER_NAME% to extract SNI as a custom header. * http: added support for :ref:`retriable health check status codes `. * http: added timing information about upstream connection and encryption establishment to stream info. These can currently be accessed via custom access loggers. +* http: added support for :ref:`forwarding HTTP1 reason phrase `. * listener: added API for extensions to access :ref:`typed_filter_metadata ` configured in the listener's :ref:`metadata ` field. * listener: added support for :ref:`MPTCP ` (multipath TCP). * listener: added support for opting out listeners from the globally set downstream connection limit via :ref:`ignore_global_conn_limit `. diff --git a/envoy/http/header_formatter.h b/envoy/http/header_formatter.h index 8615cd7a7a5d2..6a7c244e5e0a6 100644 --- a/envoy/http/header_formatter.h +++ b/envoy/http/header_formatter.h @@ -33,6 +33,16 @@ class StatefulHeaderKeyFormatter : public HeaderKeyFormatter { * Called for each header key received by the codec. */ virtual void processKey(absl::string_view key) PURE; + + /** + * Called to save received reason phrase + */ + virtual void setReasonPhrase(absl::string_view reason_phrase) PURE; + + /** + * Called to get saved reason phrase + */ + virtual absl::string_view getReasonPhrase() const PURE; }; using StatefulHeaderKeyFormatterPtr = std::unique_ptr; diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index fdd317babbbfc..0ddc9c8a69aa7 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -414,9 +414,15 @@ void ResponseEncoderImpl::encodeHeaders(const ResponseHeaderMap& headers, bool e connection_.addIntToBuffer(numeric_status); connection_.addCharToBuffer(' '); - const char* status_string = CodeUtility::toString(static_cast(numeric_status)); - uint32_t status_string_len = strlen(status_string); - connection_.copyToBuffer(status_string, status_string_len); + StatefulHeaderKeyFormatterOptConstRef formatter(headers.formatter()); + + if (formatter.has_value() && !formatter->getReasonPhrase().empty()) { + connection_.addToBuffer(formatter->getReasonPhrase()); + } else { + const char* status_string = CodeUtility::toString(static_cast(numeric_status)); + uint32_t status_string_len = strlen(status_string); + connection_.copyToBuffer(status_string, status_string_len); + } connection_.addCharToBuffer('\r'); connection_.addCharToBuffer('\n'); @@ -1301,6 +1307,16 @@ RequestEncoder& ClientConnectionImpl::newStream(ResponseDecoder& response_decode return pending_response_.value().encoder_; } +Status ClientConnectionImpl::onStatus(const char* data, size_t length) { + auto& headers = absl::get(headers_or_trailers_); + StatefulHeaderKeyFormatterOptRef formatter(headers->formatter()); + if (formatter.has_value()) { + formatter->setReasonPhrase(absl::string_view(data, length)); + } + + return okStatus(); +} + Envoy::StatusOr ClientConnectionImpl::onHeadersCompleteBase() { ENVOY_CONN_LOG(trace, "status_code {}", connection_, parser_->statusCode()); diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index c0904991cf3bf..94864dff09f0f 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -499,6 +499,7 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { // ParserCallbacks. Status onUrl(const char* data, size_t length) override; + Status onStatus(const char*, size_t) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } // ConnectionImpl void onEncodeComplete() override; StreamInfo::BytesMeter& getBytesMeter() override { @@ -593,6 +594,7 @@ class ClientConnectionImpl : public ClientConnection, public ConnectionImpl { // ParserCallbacks. Status onUrl(const char*, size_t) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + Status onStatus(const char* data, size_t length) override; // ConnectionImpl Http::Status dispatch(Buffer::Instance& data) override; void onEncodeComplete() override {} diff --git a/source/common/http/http1/legacy_parser_impl.cc b/source/common/http/http1/legacy_parser_impl.cc index 840ba419099f6..2da1d3bc177c2 100644 --- a/source/common/http/http1/legacy_parser_impl.cc +++ b/source/common/http/http1/legacy_parser_impl.cc @@ -51,7 +51,11 @@ class LegacyHttpParserImpl::Impl { auto status = conn_impl->onUrl(at, length); return conn_impl->setAndCheckCallbackStatus(std::move(status)); }, - nullptr, // on_status + [](http_parser* parser, const char* at, size_t length) -> int { + auto* conn_impl = static_cast(parser->data); + auto status = conn_impl->onStatus(at, length); + return conn_impl->setAndCheckCallbackStatus(std::move(status)); + }, [](http_parser* parser, const char* at, size_t length) -> int { auto* conn_impl = static_cast(parser->data); auto status = conn_impl->onHeaderField(at, length); diff --git a/source/common/http/http1/parser.h b/source/common/http/http1/parser.h index 23f6795a1f752..324ae41cc4149 100644 --- a/source/common/http/http1/parser.h +++ b/source/common/http/http1/parser.h @@ -72,6 +72,14 @@ class ParserCallbacks { */ virtual Status onHeaderValue(const char* data, size_t length) PURE; + /** + * Called when response status data is received. + * @param data supplies the start address. + * @param length supplies the length. + * @return Status representing success or failure. + */ + virtual Status onStatus(const char* data, size_t length) PURE; + /** * Called when headers are complete. A base routine happens first then a virtual dispatch is * invoked. Note that this only applies to headers and NOT trailers. End of diff --git a/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.cc b/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.cc index a27b35478f1e9..5e1ee269286de 100644 --- a/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.cc +++ b/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.cc @@ -4,12 +4,17 @@ #include "envoy/extensions/http/header_formatters/preserve_case/v3/preserve_case.pb.validate.h" #include "envoy/registry/registry.h" +#include "source/common/protobuf/message_validator_impl.h" + namespace Envoy { namespace Extensions { namespace Http { namespace HeaderFormatters { namespace PreserveCase { +PreserveCaseHeaderFormatter::PreserveCaseHeaderFormatter(const bool forward_reason_phrase) + : forward_reason_phrase_(forward_reason_phrase) {} + std::string PreserveCaseHeaderFormatter::format(absl::string_view key) const { const auto remembered_key_itr = original_header_keys_.find(key); // TODO(mattklein123): We can avoid string copies here if the formatter interface allowed us @@ -34,12 +39,28 @@ void PreserveCaseHeaderFormatter::processKey(absl::string_view key) { original_header_keys_.emplace(key); } +void PreserveCaseHeaderFormatter::setReasonPhrase(absl::string_view reason_phrase) { + if (forward_reason_phrase_) { + reason_phrase_ = std::string(reason_phrase); + } +}; + +absl::string_view PreserveCaseHeaderFormatter::getReasonPhrase() const { + return absl::string_view(reason_phrase_); +}; + class PreserveCaseFormatterFactory : public Envoy::Http::StatefulHeaderKeyFormatterFactory { public: + PreserveCaseFormatterFactory(const bool forward_reason_phrase) + : forward_reason_phrase_(forward_reason_phrase) {} + // Envoy::Http::StatefulHeaderKeyFormatterFactory Envoy::Http::StatefulHeaderKeyFormatterPtr create() override { - return std::make_unique(); + return std::make_unique(forward_reason_phrase_); } + +private: + const bool forward_reason_phrase_; }; class PreserveCaseFormatterFactoryConfig @@ -47,10 +68,17 @@ class PreserveCaseFormatterFactoryConfig public: // Envoy::Http::StatefulHeaderKeyFormatterFactoryConfig std::string name() const override { return "preserve_case"; } + Envoy::Http::StatefulHeaderKeyFormatterFactorySharedPtr - createFromProto(const Protobuf::Message&) override { - return std::make_shared(); + createFromProto(const Protobuf::Message& message) override { + auto config = + MessageUtil::downcastAndValidate( + message, ProtobufMessage::getStrictValidationVisitor()); + + return std::make_shared(config.forward_reason_phrase()); } + ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique(); diff --git a/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.h b/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.h index d2e3f7da00ff7..07ef7747d039d 100644 --- a/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.h +++ b/source/extensions/http/header_formatters/preserve_case/preserve_case_formatter.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/extensions/http/header_formatters/preserve_case/v3/preserve_case.pb.h" #include "envoy/http/header_formatter.h" #include "source/common/common/utility.h" @@ -13,11 +14,17 @@ namespace PreserveCase { class PreserveCaseHeaderFormatter : public Envoy::Http::StatefulHeaderKeyFormatter { public: // Envoy::Http::StatefulHeaderKeyFormatter + PreserveCaseHeaderFormatter(const bool forward_reason_phrase); + std::string format(absl::string_view key) const override; void processKey(absl::string_view key) override; + void setReasonPhrase(absl::string_view reason_phrase) override; + absl::string_view getReasonPhrase() const override; private: StringUtil::CaseUnorderedSet original_header_keys_; + bool forward_reason_phrase_{false}; + std::string reason_phrase_; }; } // namespace PreserveCase diff --git a/test/extensions/http/header_formatters/preserve_case/BUILD b/test/extensions/http/header_formatters/preserve_case/BUILD index f3998938cc5e3..d8e102c206a17 100644 --- a/test/extensions/http/header_formatters/preserve_case/BUILD +++ b/test/extensions/http/header_formatters/preserve_case/BUILD @@ -35,3 +35,17 @@ envoy_extension_cc_test( "//test/integration:http_integration_lib", ], ) + +envoy_extension_cc_test( + name = "preserve_case_formatter_reason_phrase_integration_test", + srcs = [ + "preserve_case_formatter_reason_phrase_integration_test.cc", + ], + extension_names = ["envoy.http.stateful_header_formatters.preserve_case"], + deps = [ + "//source/extensions/http/header_formatters/preserve_case:preserve_case_formatter", + "//test/integration:http_integration_lib", + "//test/integration:http_protocol_integration_lib", + "@envoy_api//envoy/extensions/http/header_formatters/preserve_case/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_reason_phrase_integration_test.cc b/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_reason_phrase_integration_test.cc new file mode 100644 index 0000000000000..d98a04a95e4e2 --- /dev/null +++ b/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_reason_phrase_integration_test.cc @@ -0,0 +1,93 @@ +#include "envoy/extensions/http/header_formatters/preserve_case/v3/preserve_case.pb.h" + +#include "test/integration/filters/common.h" +#include "test/integration/http_integration.h" +#include "test/test_common/registry.h" + +namespace Envoy { +namespace { + +struct TestParams { + Network::Address::IpVersion ip_version; + bool forward_reason_phrase; +}; + +std::string testParamsToString(const ::testing::TestParamInfo& p) { + return fmt::format("{}_{}", + p.param.ip_version == Network::Address::IpVersion::v4 ? "IPv4" : "IPv6", + p.param.forward_reason_phrase ? "enabled" : "disabled"); +} + +std::vector getTestsParams() { + std::vector ret; + + for (auto ip_version : TestEnvironment::getIpVersionsForTest()) { + ret.push_back(TestParams{ip_version, true}); + ret.push_back(TestParams{ip_version, false}); + } + + return ret; +} + +class PreserveCaseFormatterReasonPhraseIntegrationTest : public testing::TestWithParam, + public HttpIntegrationTest { +public: + PreserveCaseFormatterReasonPhraseIntegrationTest() + : HttpIntegrationTest(Http::CodecType::HTTP1, GetParam().ip_version) {} + + void SetUp() override { + setDownstreamProtocol(Http::CodecType::HTTP1); + setUpstreamProtocol(Http::CodecType::HTTP1); + } + + void initialize() override { + if (upstreamProtocol() == Http::CodecType::HTTP1) { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + ConfigHelper::HttpProtocolOptions protocol_options; + auto typed_extension_config = protocol_options.mutable_explicit_http_config() + ->mutable_http_protocol_options() + ->mutable_header_key_format() + ->mutable_stateful_formatter(); + typed_extension_config->set_name("preserve_case"); + + auto config = + TestUtility::parseYaml(fmt::format( + "forward_reason_phrase: {}", GetParam().forward_reason_phrase ? "true" : "false")); + typed_extension_config->mutable_typed_config()->PackFrom(config); + + ConfigHelper::setProtocolOptions(*bootstrap.mutable_static_resources()->mutable_clusters(0), + protocol_options); + }); + } + + HttpIntegrationTest::initialize(); + } +}; + +INSTANTIATE_TEST_SUITE_P(CaseFormatter, PreserveCaseFormatterReasonPhraseIntegrationTest, + testing::ValuesIn(getTestsParams()), testParamsToString); + +TEST_P(PreserveCaseFormatterReasonPhraseIntegrationTest, VerifyReasonPhraseEnabled) { + initialize(); + + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("http")); + auto request = "GET / HTTP/1.1\r\nhost: host\r\n\r\n"; + ASSERT_TRUE(tcp_client->write(request, false)); + + Envoy::FakeRawConnectionPtr upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(upstream_connection)); + + auto response = "HTTP/1.1 503 Slow Down\r\ncontent-length: 0\r\n\r\n"; + ASSERT_TRUE(upstream_connection->write(response)); + + auto expected_reason_phrase = + GetParam().forward_reason_phrase ? "Slow Down" : "Service Unavailable"; + // Verify that the downstream response has proper reason phrase + tcp_client->waitForData(fmt::format("HTTP/1.1 503 {}", expected_reason_phrase), false); + + tcp_client->close(); +} + +} // namespace +} // namespace Envoy diff --git a/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_test.cc b/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_test.cc index 053425ec8835c..ab24ed9de9cef 100644 --- a/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_test.cc +++ b/test/extensions/http/header_formatters/preserve_case/preserve_case_formatter_test.cc @@ -9,7 +9,7 @@ namespace HeaderFormatters { namespace PreserveCase { TEST(PreserveCaseFormatterTest, All) { - PreserveCaseHeaderFormatter formatter; + PreserveCaseHeaderFormatter formatter(false); formatter.processKey("Foo"); formatter.processKey("Bar"); formatter.processKey("BAR"); @@ -22,6 +22,22 @@ TEST(PreserveCaseFormatterTest, All) { EXPECT_EQ("baz", formatter.format("baz")); } +TEST(PreserveCaseFormatterTest, ReasonPhraseEnabled) { + PreserveCaseHeaderFormatter formatter(true); + + formatter.setReasonPhrase(absl::string_view("Slow Down")); + + EXPECT_EQ("Slow Down", formatter.getReasonPhrase()); +} + +TEST(PreserveCaseFormatterTest, ReasonPhraseDisabled) { + PreserveCaseHeaderFormatter formatter(false); + + formatter.setReasonPhrase(absl::string_view("Slow Down")); + + EXPECT_TRUE(formatter.getReasonPhrase().empty()); +} + } // namespace PreserveCase } // namespace HeaderFormatters } // namespace Http From c716ec618a69dac1ac4fd7f434e634a70de34418 Mon Sep 17 00:00:00 2001 From: Rohit Agrawal Date: Tue, 30 Nov 2021 11:32:26 -0500 Subject: [PATCH 35/51] access_log: Added new command operator %VIRTUAL_CLUSTER_NAME% to retrieve the matched VC name (#18941) Signed-off-by: Rohit Agrawal --- .../observability/access_log/usage.rst | 7 +++++++ docs/root/version_history/current.rst | 1 + envoy/router/router.h | 5 +++++ envoy/stream_info/stream_info.h | 10 ++++++++++ .../common/formatter/substitution_formatter.cc | 6 ++++++ source/common/router/config_impl.cc | 2 +- source/common/router/config_impl.h | 14 +++++++++----- source/common/router/router.cc | 3 +++ source/common/stream_info/stream_info_impl.h | 11 +++++++++++ .../formatter/substitution_formatter_test.cc | 16 ++++++++++++++++ test/common/router/router_test.cc | 6 ++++++ test/common/stream_info/test_util.h | 5 +++++ test/mocks/router/mocks.h | 2 ++ test/mocks/stream_info/mocks.cc | 5 +++++ test/mocks/stream_info/mocks.h | 4 ++++ 15 files changed, 91 insertions(+), 6 deletions(-) diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index c00a5a5c35b47..e796e65e7c3a5 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -384,6 +384,13 @@ The following command operators are supported: %ROUTE_NAME% Name of the route. +%VIRTUAL_CLUSTER_NAME% + HTTP*/gRPC + Name of the matched Virtual Cluster (if any). + + TCP/UDP + Not implemented ("-") + %UPSTREAM_HOST% Upstream host URL (e.g., tcp://ip:port for TCP connections). diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index ddaed36ae2dcd..18c19d890f335 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -56,6 +56,7 @@ Removed Config or Runtime New Features ------------ * access log: added :ref:`grpc_stream_retry_policy ` to the gRPC logger to reconnect when a connection fails to be established. +* access_log: added new access_log command operator ``%VIRTUAL_CLUSTER_NAME%`` to retrieve the matched Virtual Cluster name. * api: added support for *xds.type.v3.TypedStruct* in addition to the now-deprecated *udpa.type.v1.TypedStruct* proto message, which is a wrapper proto used to encode typed JSON data in a *google.protobuf.Any* field. * aws_request_signing_filter: added :ref:`match_excluded_headers ` to the signing filter to optionally exclude request headers from signing. * bootstrap: added :ref:`typed_dns_resolver_config ` in the bootstrap to support DNS resolver as an extension. diff --git a/envoy/router/router.h b/envoy/router/router.h index 31bcd119de35d..09cc4040943ab 100644 --- a/envoy/router/router.h +++ b/envoy/router/router.h @@ -501,6 +501,11 @@ class VirtualCluster { public: virtual ~VirtualCluster() = default; + /** + * @return the string name of the virtual cluster. + */ + virtual const absl::optional& name() const PURE; + /** * @return the stat-name of the virtual cluster. */ diff --git a/envoy/stream_info/stream_info.h b/envoy/stream_info/stream_info.h index 9836510f861a6..631a4d254eefa 100644 --- a/envoy/stream_info/stream_info.h +++ b/envoy/stream_info/stream_info.h @@ -466,6 +466,16 @@ class StreamInfo { */ virtual const std::string& getRouteName() const PURE; + /** + * @param std::string name denotes the name of the virtual cluster. + */ + virtual void setVirtualClusterName(const absl::optional& name) PURE; + + /** + * @return std::string& the name of the virtual cluster which got matched. + */ + virtual const absl::optional& virtualClusterName() const PURE; + /** * @param bytes_received denotes number of bytes to add to total received bytes. */ diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index fb0cba19ff945..cb78953df3650 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -686,6 +686,7 @@ class StreamInfoSslConnectionInfoFieldExtractor : public StreamInfoFormatter::Fi }; StreamInfoFormatter::StreamInfoFormatter(const std::string& field_name) { + // TODO: Change this huge if-else ladder to use a switch case instead. if (field_name == "REQUEST_DURATION") { field_extractor_ = std::make_unique( [](const StreamInfo::StreamInfo& stream_info) { @@ -953,6 +954,11 @@ StreamInfoFormatter::StreamInfoFormatter(const std::string& field_name) { } return absl::nullopt; }); + } else if (field_name == "VIRTUAL_CLUSTER_NAME") { + field_extractor_ = std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) -> absl::optional { + return stream_info.virtualClusterName(); + }); } else if (field_name == "TLS_JA3_FINGERPRINT") { field_extractor_ = std::make_unique( [](const StreamInfo::StreamInfo& stream_info) { diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index e8170860523bf..d3120fcb533ba 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -1430,7 +1430,7 @@ VirtualHostImpl::VirtualClusterEntry::VirtualClusterEntry( const envoy::config::route::v3::VirtualCluster& virtual_cluster, Stats::Scope& scope, const VirtualClusterStatNames& stat_names) : StatNameProvider(virtual_cluster.name(), scope.symbolTable()), - VirtualClusterBase(stat_name_storage_.statName(), + VirtualClusterBase(virtual_cluster.name(), stat_name_storage_.statName(), scope.scopeFromStatName(stat_name_storage_.statName()), stat_names) { if (virtual_cluster.headers().empty()) { throw EnvoyException("virtual clusters must define 'headers'"); diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 6b9aac04a25fe..e6402afc3b4df 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -236,16 +236,20 @@ class VirtualHostImpl : public VirtualHost, Logger::Loggable struct VirtualClusterBase : public VirtualCluster { public: - VirtualClusterBase(Stats::StatName stat_name, Stats::ScopePtr&& scope, - const VirtualClusterStatNames& stat_names) - : stat_name_(stat_name), scope_(std::move(scope)), + VirtualClusterBase(const absl::optional& name, Stats::StatName stat_name, + Stats::ScopePtr&& scope, const VirtualClusterStatNames& stat_names) + : name_(name), stat_name_(stat_name), scope_(std::move(scope)), stats_(generateStats(*scope_, stat_names)) {} // Router::VirtualCluster + // name_ and stat_name_ are two different representations for the same string, retained in + // memory to avoid symbol-table locks that would be needed when converting on-the-fly. + const absl::optional& name() const override { return name_; } Stats::StatName statName() const override { return stat_name_; } VirtualClusterStats& stats() const override { return stats_; } private: + const absl::optional name_; const Stats::StatName stat_name_; Stats::ScopePtr scope_; mutable VirtualClusterStats stats_; @@ -259,8 +263,8 @@ class VirtualHostImpl : public VirtualHost, Logger::Loggable struct CatchAllVirtualCluster : public VirtualClusterBase { CatchAllVirtualCluster(Stats::Scope& scope, const VirtualClusterStatNames& stat_names) - : VirtualClusterBase(stat_names.other_, scope.scopeFromStatName(stat_names.other_), - stat_names) {} + : VirtualClusterBase(absl::nullopt, stat_names.other_, + scope.scopeFromStatName(stat_names.other_), stat_names) {} }; static const std::shared_ptr SSL_REDIRECT_ROUTE; diff --git a/source/common/router/router.cc b/source/common/router/router.cc index 59336369ab11d..98bec7fb9b8df 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -453,6 +453,9 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, // Set up stat prefixes, etc. request_vcluster_ = route_entry_->virtualCluster(headers); + if (request_vcluster_ != nullptr) { + callbacks_->streamInfo().setVirtualClusterName(request_vcluster_->name()); + } ENVOY_STREAM_LOG(debug, "cluster '{}' match for URL '{}'", *callbacks_, route_entry_->clusterName(), headers.getPathValue()); diff --git a/source/common/stream_info/stream_info_impl.h b/source/common/stream_info/stream_info_impl.h index 0077ace81f281..9fdc38fb6d39f 100644 --- a/source/common/stream_info/stream_info_impl.h +++ b/source/common/stream_info/stream_info_impl.h @@ -292,6 +292,14 @@ struct StreamInfoImpl : public StreamInfo { const std::string& getRouteName() const override { return route_name_; } + void setVirtualClusterName(const absl::optional& virtual_cluster_name) override { + virtual_cluster_name_ = virtual_cluster_name; + } + + const absl::optional& virtualClusterName() const override { + return virtual_cluster_name_; + } + void setUpstreamLocalAddress( const Network::Address::InstanceConstSharedPtr& upstream_local_address) override { maybeCreateUpstreamInfo(); @@ -443,6 +451,9 @@ struct StreamInfoImpl : public StreamInfo { FilterStateSharedPtr legacy_upstream_filter_state_; std::string route_name_; absl::optional attempt_count_; + // TODO(agrawroh): Check if the owner of this storage outlives the StreamInfo. We should only copy + // the string if it could outlive the StreamInfo. + absl::optional virtual_cluster_name_; private: static Network::ConnectionInfoProviderSharedPtr emptyDownstreamAddressProvider() { diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 6753aff7d7117..8cb401f900fcc 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -752,6 +752,22 @@ TEST(SubstitutionFormatterTest, streamInfoFormatterWithSsl) { Http::TestResponseTrailerMapImpl response_trailers; std::string body; + { + NiceMock stream_info; + StreamInfoFormatter upstream_format("VIRTUAL_CLUSTER_NAME"); + std::string virtual_cluster_name = "authN"; + stream_info.setVirtualClusterName(virtual_cluster_name); + EXPECT_EQ("authN", upstream_format.format(request_headers, response_headers, response_trailers, + stream_info, body)); + } + + { + NiceMock stream_info; + StreamInfoFormatter upstream_format("VIRTUAL_CLUSTER_NAME"); + EXPECT_EQ(absl::nullopt, upstream_format.format(request_headers, response_headers, + response_trailers, stream_info, body)); + } + { // Use a local stream info for these tests as as setSslConnection can only be called once. NiceMock stream_info; diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 46506b71174a4..ae008d6228ee5 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -5912,6 +5912,9 @@ TEST_F(RouterTest, CanaryStatusTrue) { Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-alt-stat-name", "alt_stat"}, {"x-envoy-internal", "true"}}; HttpTestUtility::addDefaultHeaders(headers); + const absl::optional virtual_cluster_name = + absl::optional("fake_virtual_cluster"); + EXPECT_CALL(callbacks_.stream_info_, setVirtualClusterName(virtual_cluster_name)); router_.decodeHeaders(headers, true); EXPECT_EQ(1U, callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); @@ -5949,6 +5952,9 @@ TEST_F(RouterTest, CanaryStatusFalse) { Http::TestRequestHeaderMapImpl headers{{"x-envoy-upstream-alt-stat-name", "alt_stat"}, {"x-envoy-internal", "true"}}; HttpTestUtility::addDefaultHeaders(headers); + const absl::optional virtual_cluster_name = + absl::optional("fake_virtual_cluster"); + EXPECT_CALL(callbacks_.stream_info_, setVirtualClusterName(virtual_cluster_name)); router_.decodeHeaders(headers, true); EXPECT_EQ(1U, callbacks_.route_->route_entry_.virtual_cluster_.stats().upstream_rq_total_.value()); diff --git a/test/common/stream_info/test_util.h b/test/common/stream_info/test_util.h index adad891693ba4..903bf7221b91f 100644 --- a/test/common/stream_info/test_util.h +++ b/test/common/stream_info/test_util.h @@ -34,6 +34,10 @@ class TestStreamInfo : public StreamInfo::StreamInfoImpl { return *downstream_connection_info_provider_; } + const absl::optional& virtualClusterName() const override { + return virtual_cluster_name_; + } + void onRequestComplete() override { end_time_ = timeSystem().monotonicTime(); } absl::optional requestComplete() const override { @@ -54,6 +58,7 @@ class TestStreamInfo : public StreamInfo::StreamInfoImpl { SystemTime start_time_; MonotonicTime start_time_monotonic_; absl::optional end_time_; + absl::optional virtual_cluster_name_; Network::ConnectionInfoSetterSharedPtr downstream_connection_info_provider_{ std::make_shared(nullptr, nullptr)}; Envoy::Event::SimulatedTimeSystem test_time_; diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index e5684f335842c..9942d0d936991 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -263,9 +263,11 @@ class MockShadowWriter : public ShadowWriter { class TestVirtualCluster : public VirtualCluster { public: // Router::VirtualCluster + const absl::optional& name() const override { return name_; } Stats::StatName statName() const override { return stat_name_.statName(); } VirtualClusterStats& stats() const override { return stats_; } + const absl::optional name_ = "fake_virtual_cluster"; Stats::TestUtil::TestSymbolTable symbol_table_; Stats::StatNameManagedStorage stat_name_{"fake_virtual_cluster", *symbol_table_}; Stats::IsolatedStoreImpl stats_store_; diff --git a/test/mocks/stream_info/mocks.cc b/test/mocks/stream_info/mocks.cc index 3bc5473c55385..fa37db802b260 100644 --- a/test/mocks/stream_info/mocks.cc +++ b/test/mocks/stream_info/mocks.cc @@ -110,7 +110,12 @@ MockStreamInfo::MockStreamInfo() ON_CALL(*this, setRouteName(_)).WillByDefault(Invoke([this](const absl::string_view route_name) { route_name_ = std::string(route_name); })); + ON_CALL(*this, setVirtualClusterName(_)) + .WillByDefault(Invoke([this](const absl::optional& virtual_cluster_name) { + virtual_cluster_name_ = virtual_cluster_name; + })); ON_CALL(*this, getRouteName()).WillByDefault(ReturnRef(route_name_)); + ON_CALL(*this, virtualClusterName()).WillByDefault(ReturnRef(virtual_cluster_name_)); ON_CALL(*this, upstreamTransportFailureReason()) .WillByDefault(ReturnRef(upstream_transport_failure_reason_)); ON_CALL(*this, setConnectionID(_)).WillByDefault(Invoke([this](uint64_t id) { diff --git a/test/mocks/stream_info/mocks.h b/test/mocks/stream_info/mocks.h index 4f96a1d16ff4b..485a4c7244df5 100644 --- a/test/mocks/stream_info/mocks.h +++ b/test/mocks/stream_info/mocks.h @@ -54,7 +54,10 @@ class MockStreamInfo : public StreamInfo { MOCK_METHOD(void, addWireBytesReceived, (uint64_t)); MOCK_METHOD(uint64_t, wireBytesReceived, (), (const)); MOCK_METHOD(void, setRouteName, (absl::string_view route_name)); + MOCK_METHOD(void, setVirtualClusterName, + (const absl::optional& virtual_cluster_name)); MOCK_METHOD(const std::string&, getRouteName, (), (const)); + MOCK_METHOD(const absl::optional&, virtualClusterName, (), (const)); MOCK_METHOD(absl::optional, protocol, (), (const)); MOCK_METHOD(void, protocol, (Http::Protocol protocol)); MOCK_METHOD(absl::optional, responseCode, (), (const)); @@ -145,6 +148,7 @@ class MockStreamInfo : public StreamInfo { std::string filter_chain_name_; absl::optional upstream_connection_id_; absl::optional attempt_count_; + absl::optional virtual_cluster_name_; DownstreamTiming downstream_timing_; }; From 7ee1f77a6f52f5215343c623fefb33067578016b Mon Sep 17 00:00:00 2001 From: Misha Efimov Date: Tue, 30 Nov 2021 11:37:23 -0500 Subject: [PATCH 36/51] Remove wrong ASSERT from MockOsSysCalls::setsockopt(). (#19126) Signed-off-by: Misha Efimov --- test/mocks/api/mocks.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/mocks/api/mocks.cc b/test/mocks/api/mocks.cc index c3eeed40d0c5c..6a396cd4e064c 100644 --- a/test/mocks/api/mocks.cc +++ b/test/mocks/api/mocks.cc @@ -60,14 +60,14 @@ MockOsSysCalls::~MockOsSysCalls() = default; SysCallIntResult MockOsSysCalls::setsockopt(os_fd_t sockfd, int level, int optname, const void* optval, socklen_t optlen) { - ASSERT(optlen == sizeof(int)); - // Allow mocking system call failure. if (setsockopt_(sockfd, level, optname, optval, optlen) != 0) { return SysCallIntResult{-1, 0}; } - boolsockopts_[SockOptKey(sockfd, level, optname)] = !!*reinterpret_cast(optval); + if (optlen >= sizeof(int)) { + boolsockopts_[SockOptKey(sockfd, level, optname)] = !!*reinterpret_cast(optval); + } return SysCallIntResult{0, 0}; }; From 3baae860b020d0014938f061c2fa0b215ecbc1df Mon Sep 17 00:00:00 2001 From: Yao Zengzeng Date: Wed, 1 Dec 2021 00:37:42 +0800 Subject: [PATCH 37/51] http2: remove unnecessary negative_capacity_ (#19057) Commit Message: remove negative_capacity_ to make the code easier to understand Additional Description: fixes part I of #18880 Risk Level: high Testing: unit Docs Changes: n/a Signed-off-by: YaoZengzeng --- source/common/conn_pool/conn_pool_base.cc | 21 +++++----- source/common/http/conn_pool_base.cc | 1 - source/common/http/conn_pool_base.h | 13 ------ test/common/http/http2/conn_pool_test.cc | 49 +++++++++++++++++++++-- tools/spelling/spelling_dictionary.txt | 1 + 5 files changed, 58 insertions(+), 27 deletions(-) diff --git a/source/common/conn_pool/conn_pool_base.cc b/source/common/conn_pool/conn_pool_base.cc index 11b65b5555866..9850addcc30a5 100644 --- a/source/common/conn_pool/conn_pool_base.cc +++ b/source/common/conn_pool/conn_pool_base.cc @@ -208,22 +208,25 @@ void ConnPoolImplBase::onStreamClosed(Envoy::ConnectionPool::ActiveClient& clien bool delay_attaching_stream) { ENVOY_CONN_LOG(debug, "destroying stream: {} remaining", client, client.numActiveStreams()); ASSERT(num_active_streams_ > 0); - // Reflect there's one less stream in flight. - bool had_negative_capacity = client.hadNegativeDeltaOnStreamClosed(); state_.decrActiveStreams(1); num_active_streams_--; host_->stats().rq_active_.dec(); host_->cluster().stats().upstream_rq_active_.dec(); host_->cluster().resourceManager(priority_).requests().dec(); - // If the effective client capacity was limited by concurrency, increase connecting capacity. - // If the effective client capacity was limited by max total streams, this will not result in an - // increment as no capacity is freed up. // We don't update the capacity for HTTP/3 as the stream count should only // increase when a MAX_STREAMS frame is received. - if (trackStreamCapacity() && (client.remaining_streams_ > client.concurrent_stream_limit_ - - client.numActiveStreams() - 1 || - had_negative_capacity)) { - state_.incrConnectingAndConnectedStreamCapacity(1); + if (trackStreamCapacity()) { + // If the effective client capacity was limited by concurrency, increase connecting capacity. + bool limited_by_concurrency = + client.remaining_streams_ > client.concurrent_stream_limit_ - client.numActiveStreams() - 1; + // The capacity calculated by concurrency could be negative if a SETTINGS frame lowered the + // number of allowed streams. In this case, effective client capacity was still limited by + // concurrency, compare client.concurrent_stream_limit_ and client.numActiveStreams() directly + // to avoid overflow. + bool negative_capacity = client.concurrent_stream_limit_ < client.numActiveStreams() + 1; + if (negative_capacity || limited_by_concurrency) { + state_.incrConnectingAndConnectedStreamCapacity(1); + } } if (client.state() == ActiveClient::State::DRAINING && client.numActiveStreams() == 0) { // Close out the draining client if we no longer have active streams. diff --git a/source/common/http/conn_pool_base.cc b/source/common/http/conn_pool_base.cc index e43b369c0b01c..4f823caf777c9 100644 --- a/source/common/http/conn_pool_base.cc +++ b/source/common/http/conn_pool_base.cc @@ -127,7 +127,6 @@ void MultiplexedActiveClientBase::onSettings(ReceivedSettings& settings) { } parent_.decrClusterStreamCapacity(delta); ENVOY_CONN_LOG(trace, "Decreasing stream capacity by {}", *codec_client_, delta); - negative_capacity_ += delta; } // As we don't increase stream limits when maxConcurrentStreams goes up, treat // a stream limit of 0 as a GOAWAY. diff --git a/source/common/http/conn_pool_base.h b/source/common/http/conn_pool_base.h index 47bdfdad7f3f2..e3bdd93928f52 100644 --- a/source/common/http/conn_pool_base.h +++ b/source/common/http/conn_pool_base.h @@ -201,19 +201,6 @@ class MultiplexedActiveClientBase : public CodecClientCallbacks, void onGoAway(Http::GoAwayErrorCode error_code) override; void onSettings(ReceivedSettings& settings) override; - // As this is called once when the stream is closed, it's a good place to - // update the counter as one stream has been "returned" and the negative - // capacity should be reduced. - bool hadNegativeDeltaOnStreamClosed() override { - int ret = negative_capacity_ != 0; - if (negative_capacity_ > 0) { - negative_capacity_--; - } - return ret; - } - - uint64_t negative_capacity_{}; - protected: MultiplexedActiveClientBase(Envoy::Http::HttpConnPoolImplBase& parent, Upstream::Host::CreateConnectionData& data, diff --git a/test/common/http/http2/conn_pool_test.cc b/test/common/http/http2/conn_pool_test.cc index aaf674207d974..5b917ea935fb7 100644 --- a/test/common/http/http2/conn_pool_test.cc +++ b/test/common/http/http2/conn_pool_test.cc @@ -1669,6 +1669,47 @@ TEST_F(Http2ConnPoolImplTest, MaybePreconnect) { closeAllClients(); } +TEST_F(Http2ConnPoolImplTest, TestUnusedCapacity) { + cluster_->http2_options_.mutable_max_concurrent_streams()->set_value(8); + cluster_->max_requests_per_connection_ = 6; + + expectClientsCreate(1); + ActiveTestRequest r1(*this, 0, false); + // Initially, capacity is based on remaining streams and capped at 6. + CHECK_STATE(0 /*active*/, 1 /*pending*/, 6 /*capacity*/); + expectClientConnect(0, r1); + // Now the stream is active, remaining concurrency capacity is 5. + CHECK_STATE(1 /*active*/, 0 /*pending*/, 5 /*capacity*/); + + // With two more streams, remaining unused capacity is 3. + ActiveTestRequest r2(*this, 0, true); + ActiveTestRequest r3(*this, 0, true); + CHECK_STATE(3 /*active*/, 0 /*pending*/, 3 /*capacity*/); + + // Settings frame results in 1 unused capacity. + NiceMock settings; + settings.max_concurrent_streams_ = 4; + test_clients_[0].codec_client_->onSettings(settings); + CHECK_STATE(3 /*active*/, 0 /*pending*/, 1 /*capacity*/); + + // Closing a stream, unused capacity returns to 2. + completeRequest(r1); + CHECK_STATE(2 /*active*/, 0 /*pending*/, 2 /*capacity*/); + + // Closing another, unused capacity returns to 3 (3 remaining stream). + completeRequest(r2); + CHECK_STATE(1 /*active*/, 0 /*pending*/, 3 /*capacity*/); + + // Closing the last stream, unused capacity remains at 3, as there is only 3 remaining streams. + completeRequest(r3); + CHECK_STATE(0 /*active*/, 0 /*pending*/, 3 /*capacity*/); + + // Clean up with an outstanding stream. + pool_->drainConnections(Envoy::ConnectionPool::DrainBehavior::DrainExistingConnections); + closeAllClients(); + CHECK_STATE(0 /*active*/, 0 /*pending*/, 0 /*capacity*/); +} + TEST_F(Http2ConnPoolImplTest, TestStateWithMultiplexing) { cluster_->http2_options_.mutable_max_concurrent_streams()->set_value(2); cluster_->max_requests_per_connection_ = 4; @@ -1678,22 +1719,22 @@ TEST_F(Http2ConnPoolImplTest, TestStateWithMultiplexing) { // Initially, capacity is based on concurrency and capped at 2. CHECK_STATE(0 /*active*/, 1 /*pending*/, 2 /*capacity*/); expectClientConnect(0, r1); - // Now the stream is active, remaining concurrency capacity is 1 + // Now the stream is active, remaining concurrency capacity is 1. CHECK_STATE(1 /*active*/, 0 /*pending*/, 1 /*capacity*/); // With one more stream, remaining concurrency capacity is 0. ActiveTestRequest r2(*this, 0, true); CHECK_STATE(2 /*active*/, 0 /*pending*/, 0 /*capacity*/); - // If one stream closes, concurrency capacity goes to 1 (2 remaining streams) + // If one stream closes, concurrency capacity goes to 1 (2 remaining streams). completeRequest(r1); CHECK_STATE(1 /*active*/, 0 /*pending*/, 1 /*capacity*/); - // Assigning a new stream, concurrency capacity returns to 0 (1 remaining stream); + // Assigning a new stream, concurrency capacity returns to 0 (1 remaining stream). ActiveTestRequest r3(*this, 0, true); CHECK_STATE(2 /*active*/, 0 /*pending*/, 0 /*capacity*/); - // Closing a stream, capacity returns to 1 (both concurrency and remaining streams) + // Closing a stream, capacity returns to 1 (both concurrency and remaining streams). completeRequest(r2); CHECK_STATE(1 /*active*/, 0 /*pending*/, 1 /*capacity*/); diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 0e4e9aaf1e3e1..180ecf3fa528a 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -868,6 +868,7 @@ nan nanos natively ndk +negative netblock netblocks netfilter From a02d3ce726aba2b74e8c6b4a8ef7554a99416d36 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 1 Dec 2021 10:08:28 -0500 Subject: [PATCH 38/51] http: testing failed writes (#19021) Risk Level: n/a (test only) Testing: new integration test Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- source/common/network/io_socket_error_impl.cc | 6 +++++ source/common/network/io_socket_error_impl.h | 2 ++ test/integration/BUILD | 1 + test/integration/protocol_integration_test.cc | 22 +++++++++++++++++++ test/integration/socket_interface_swap.cc | 8 +++---- test/integration/socket_interface_swap.h | 20 ++++++++++------- 6 files changed, 47 insertions(+), 12 deletions(-) diff --git a/source/common/network/io_socket_error_impl.cc b/source/common/network/io_socket_error_impl.cc index a3e955f4d68a2..b2a4e93f9e726 100644 --- a/source/common/network/io_socket_error_impl.cc +++ b/source/common/network/io_socket_error_impl.cc @@ -18,6 +18,12 @@ IoSocketError* IoSocketError::getIoSocketInvalidAddressInstance() { return instance; } +IoSocketError* IoSocketError::getIoSocketEbadfInstance() { + static auto* instance = + new IoSocketError(SOCKET_ERROR_BADF, Api::IoError::IoErrorCode::NoSupport); + return instance; +} + IoSocketError* IoSocketError::getIoSocketEagainInstance() { static auto* instance = new IoSocketError(SOCKET_ERROR_AGAIN, Api::IoError::IoErrorCode::Again); return instance; diff --git a/source/common/network/io_socket_error_impl.h b/source/common/network/io_socket_error_impl.h index d17ce2a2f31cf..392dd6129ebba 100644 --- a/source/common/network/io_socket_error_impl.h +++ b/source/common/network/io_socket_error_impl.h @@ -25,6 +25,8 @@ class IoSocketError : public Api::IoError { // deleter deleteIoError() below to avoid deallocating memory for this error. static IoSocketError* getIoSocketEagainInstance(); + static IoSocketError* getIoSocketEbadfInstance(); + // This error is introduced when Envoy create socket for unsupported address. It is either a bug, // or this Envoy instance received config which is not yet supported. This should not be fatal // error. diff --git a/test/integration/BUILD b/test/integration/BUILD index 23d783d586501..a1348059a050b 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -501,6 +501,7 @@ envoy_cc_test_library( ], deps = [ ":http_protocol_integration_lib", + ":socket_interface_swap_lib", "//source/common/http:header_map_lib", "//source/extensions/filters/http/buffer:config", "//test/common/http/http2:http2_frame", diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index c43ac822ee2bc..a35ee6e29eb03 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -30,6 +30,7 @@ #include "test/common/upstream/utility.h" #include "test/integration/autonomous_upstream.h" #include "test/integration/http_integration.h" +#include "test/integration/socket_interface_swap.h" #include "test/integration/test_host_predicate_config.h" #include "test/integration/utility.h" #include "test/mocks/upstream/retry_priority.h" @@ -3533,4 +3534,25 @@ TEST_P(DownstreamProtocolIntegrationTest, ContentLengthLargerThanPayload) { EXPECT_EQ(Http::StreamResetReason::RemoteReset, response->resetReason()); } +TEST_P(DownstreamProtocolIntegrationTest, HandleSocketFail) { + SocketInterfaceSwap socket_swap; + if (downstreamProtocol() == Http::CodecType::HTTP3) { + return; + } + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(); + + // Makes us have Envoy's writes to downstream return EBADF + Network::IoSocketError* ebadf = Network::IoSocketError::getIoSocketEbadfInstance(); + socket_swap.writev_matcher_->setSourcePort(lookupPort("http")); + socket_swap.writev_matcher_->setWritevOverride(ebadf); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); + + ASSERT_TRUE(codec_client_->waitForDisconnect()); + socket_swap.writev_matcher_->setWritevOverride(nullptr); +} + } // namespace Envoy diff --git a/test/integration/socket_interface_swap.cc b/test/integration/socket_interface_swap.cc index 521153817a256..94d8abba9fbd1 100644 --- a/test/integration/socket_interface_swap.cc +++ b/test/integration/socket_interface_swap.cc @@ -9,10 +9,10 @@ SocketInterfaceSwap::SocketInterfaceSwap() { [writev_matcher = writev_matcher_](Envoy::Network::TestIoSocketHandle* io_handle, const Buffer::RawSlice*, uint64_t) -> absl::optional { - if (writev_matcher->shouldReturnEgain(io_handle)) { + Network::IoSocketError* error_override = writev_matcher->returnOverride(io_handle); + if (error_override) { return Api::IoCallUint64Result( - 0, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)); + 0, Api::IoErrorPtr(error_override, Network::IoSocketError::deleteIoError)); } return absl::nullopt; })); @@ -23,7 +23,7 @@ void SocketInterfaceSwap::IoHandleMatcher::setResumeWrites() { mutex_.Await(absl::Condition( +[](Network::TestIoSocketHandle** matched_iohandle) { return *matched_iohandle != nullptr; }, &matched_iohandle_)); - writev_returns_egain_ = false; + error_ = nullptr; matched_iohandle_->activateInDispatcherThread(Event::FileReadyType::Write); } diff --git a/test/integration/socket_interface_swap.h b/test/integration/socket_interface_swap.h index ee4f84db47659..820041a5d4eb0 100644 --- a/test/integration/socket_interface_swap.h +++ b/test/integration/socket_interface_swap.h @@ -14,16 +14,16 @@ class SocketInterfaceSwap { // Object of this class hold the state determining the IoHandle which // should return EAGAIN from the `writev` call. struct IoHandleMatcher { - bool shouldReturnEgain(Envoy::Network::TestIoSocketHandle* io_handle) { + Network::IoSocketError* returnOverride(Envoy::Network::TestIoSocketHandle* io_handle) { absl::MutexLock lock(&mutex_); - if (writev_returns_egain_ && (io_handle->localAddress()->ip()->port() == src_port_ || - io_handle->peerAddress()->ip()->port() == dst_port_)) { + if (error_ && (io_handle->localAddress()->ip()->port() == src_port_ || + io_handle->peerAddress()->ip()->port() == dst_port_)) { ASSERT(matched_iohandle_ == nullptr || matched_iohandle_ == io_handle, "Matched multiple io_handles, expected at most one to match."); matched_iohandle_ = io_handle; - return true; + return error_; } - return false; + return nullptr; } // Source port to match. The port specified should be associated with a listener. @@ -41,9 +41,14 @@ class SocketInterfaceSwap { } void setWritevReturnsEgain() { + setWritevOverride(Network::IoSocketError::getIoSocketEagainInstance()); + } + + // The caller is responsible for memory management. + void setWritevOverride(Network::IoSocketError* error) { absl::WriterMutexLock lock(&mutex_); ASSERT(src_port_ != 0 || dst_port_ != 0); - writev_returns_egain_ = true; + error_ = error; } void setResumeWrites(); @@ -52,7 +57,7 @@ class SocketInterfaceSwap { mutable absl::Mutex mutex_; uint32_t src_port_ ABSL_GUARDED_BY(mutex_) = 0; uint32_t dst_port_ ABSL_GUARDED_BY(mutex_) = 0; - bool writev_returns_egain_ ABSL_GUARDED_BY(mutex_) = false; + Network::IoSocketError* error_ ABSL_GUARDED_BY(mutex_) = nullptr; Network::TestIoSocketHandle* matched_iohandle_{}; }; @@ -63,7 +68,6 @@ class SocketInterfaceSwap { Envoy::Network::SocketInterfaceSingleton::initialize(previous_socket_interface_); } -protected: Envoy::Network::SocketInterface* const previous_socket_interface_{ Envoy::Network::SocketInterfaceSingleton::getExisting()}; std::shared_ptr writev_matcher_{std::make_shared()}; From ccc563eb57c778c7bd32af2c10346362860b4889 Mon Sep 17 00:00:00 2001 From: Faseela K Date: Wed, 1 Dec 2021 18:42:36 +0100 Subject: [PATCH 39/51] Update boringssl version (#19152) Signed-off-by: Faseela K --- bazel/repository_locations.bzl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 6d4bac9928ee3..55e87d55300e2 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -79,15 +79,15 @@ REPOSITORY_LOCATIONS_SPEC = dict( # To update BoringSSL, which tracks Chromium releases: # 1. Open https://omahaproxy.appspot.com/ and note of linux/dev release. # 2. Open https://chromium.googlesource.com/chromium/src/+/refs/tags//DEPS and note . - # 3. Find a commit in BoringSSL's "master-with-bazel" branch that merges . + # 3. Find a commit in BoringSSL's "main-with-bazel" branch that merges . # - # chromium-92.0.4511.0 (linux/dev) - version = "75edea1922aefe415e0e60ac576116634b0a94f8", - sha256 = "70e9d8737e35d67f94b9e742ca59c02c36f30f1d822d5a3706511a23798d8049", + # chromium-94.0.4606.81 (linux/dev) + version = "648cbaf033401b7fe7acdce02f275b06a88aab5c", + sha256 = "579cb415458e9f3642da0a39a72f79fdfe6dc9c1713b3a823f1e276681b9703e", strip_prefix = "boringssl-{version}", urls = ["https://github.com/google/boringssl/archive/{version}.tar.gz"], use_category = ["controlplane", "dataplane_core"], - release_date = "2021-05-13", + release_date = "2021-07-15", cpe = "cpe:2.3:a:google:boringssl:*", ), boringssl_fips = dict( From 510f1a95ea170a35842b3ad8dd245a73b12a41d5 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Wed, 1 Dec 2021 13:30:39 -0500 Subject: [PATCH 40/51] listener: disallow duplicate port specification (#19145) This bug existed previously, but with the recent reuse port as the default change is now more obvious. Previously, we would allow multiple listeners to listen on the same port, which is obviously very wrong. This change blocks that at config load time. Fixes https://github.com/envoyproxy/envoy/issues/19099 Signed-off-by: Matt Klein --- docs/root/version_history/current.rst | 1 + source/server/listener_manager_impl.cc | 14 ++++++------ test/server/listener_manager_impl_test.cc | 28 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 18c19d890f335..9fe7a46bd3985 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -29,6 +29,7 @@ Bug Fixes * ext_authz: fix the ext_authz http filter to correctly set response flags to ``UAEX`` when a connection is denied. * ext_authz: fix the ext_authz network filter to correctly set response flag and code details to ``UAEX`` when a connection is denied. * listener: fixed the crash when updating listeners that do not bind to port. +* listener: fixed issue where more than one listener could listen on the same port if using reuse port, thus randomly accepting connections on different listeners. This configuration is now rejected. * tcp: fixed a bug where upstream circuit breakers applied HTTP per-request bounds to TCP connections. * thrift_proxy: fix the thrift_proxy connection manager to correctly report success/error response metrics when performing :ref:`payload passthrough `. diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 545645f01cc29..1bae6e6a8314d 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -961,13 +961,13 @@ Network::DrainableFilterChainSharedPtr ListenerFilterChainFactoryBuilder::buildF void ListenerManagerImpl::setNewOrDrainingSocketFactory( const std::string& name, const envoy::config::core::v3::Address& proto_address, ListenerImpl& listener) { - // Typically we catch address issues when we try to bind to the same address multiple times. - // However, for listeners that do not bind we must check to make sure we are not duplicating. This - // is an edge case and nothing will explicitly break, but there is no possibility that two - // listeners that do not bind will ever be used. Only the first one will be used when searched for - // by address. Thus we block it. - if (!listener.bindToPort() && (hasListenerWithCompatibleAddress(warming_listeners_, listener) || - hasListenerWithCompatibleAddress(active_listeners_, listener))) { + // For listeners that do not bind or listeners that do not bind to port 0 we must check to make + // sure we are not duplicating the address. This avoids ambiguity about which non-binding + // listener is used or even worse for the binding to port != 0 and reuse port case multiple + // different listeners receiving connections destined for the same port. + if ((!listener.bindToPort() || listener.config().address().socket_address().port_value() != 0) && + (hasListenerWithCompatibleAddress(warming_listeners_, listener) || + hasListenerWithCompatibleAddress(active_listeners_, listener))) { const std::string message = fmt::format("error adding listener: '{}' has duplicate address '{}' as existing listener", name, listener.address()->asString()); diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 8002094c46467..69f7166ef1295 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -206,6 +206,34 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, DefaultListenerPerConnectionBuffe EXPECT_EQ(1024 * 1024U, manager_->listeners().back().get().perConnectionBufferLimitBytes()); } +TEST_F(ListenerManagerImplWithRealFiltersTest, DuplicatePortNotAllowed) { + const std::string yaml1 = R"EOF( +name: foo +address: + socket_address: + address: 127.0.0.1 + port_value: 1234 +filter_chains: +- filters: [] + )EOF"; + + const std::string yaml2 = R"EOF( +name: bar +address: + socket_address: + address: 127.0.0.1 + port_value: 1234 +filter_chains: +- filters: [] + )EOF"; + + EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); + manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml1), "", true); + EXPECT_THROW_WITH_MESSAGE( + manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml2), "", true), EnvoyException, + "error adding listener: 'bar' has duplicate address '127.0.0.1:1234' as existing listener"); +} + TEST_F(ListenerManagerImplWithRealFiltersTest, SetListenerPerConnectionBufferLimit) { const std::string yaml = R"EOF( address: From ef1faeabbe3e1c85bcb4a3ccddb1c34aec375c7c Mon Sep 17 00:00:00 2001 From: Rohit Agrawal Date: Wed, 1 Dec 2021 16:06:58 -0500 Subject: [PATCH 41/51] access_log: added new access_log command operator %UPSTREAM_REQUEST_ATTEMPT_COUNT% (#19150) Adds a new command operator called %UPSTREAM_REQUEST_ATTEMPT_COUNT% in the access logs which can be used to retrieve the number of times given request got attempted upstream. Fixes #18870 Signed-off-by: Rohit Agrawal --- .../observability/access_log/usage.rst | 11 ++++++++++ docs/root/version_history/current.rst | 1 + .../formatter/substitution_formatter.cc | 5 +++++ .../formatter/substitution_formatter_test.cc | 22 +++++++++++++++++++ 4 files changed, 39 insertions(+) diff --git a/docs/root/configuration/observability/access_log/usage.rst b/docs/root/configuration/observability/access_log/usage.rst index e796e65e7c3a5..8599f838f4f30 100644 --- a/docs/root/configuration/observability/access_log/usage.rst +++ b/docs/root/configuration/observability/access_log/usage.rst @@ -241,6 +241,17 @@ The following command operators are supported: TCP Downstream bytes sent on connection. +%UPSTREAM_REQUEST_ATTEMPT_COUNT% + HTTP + Number of times the request is attempted upstream. Note that an attempt count of '0' means that + the request was never attempted upstream. + + TCP + Number of times the connection request is attempted upstream. Note that an attempt count of '0' + means that the connection request was never attempted upstream. + + Renders a numeric value in typed JSON logs. + %UPSTREAM_WIRE_BYTES_SENT% HTTP Total number of bytes sent to the upstream by the http stream. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 9fe7a46bd3985..76c77a7e55d36 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -57,6 +57,7 @@ Removed Config or Runtime New Features ------------ * access log: added :ref:`grpc_stream_retry_policy ` to the gRPC logger to reconnect when a connection fails to be established. +* access_log: added new access_log command operator ``%UPSTREAM_REQUEST_ATTEMPT_COUNT%`` to retrieve the number of times given request got attempted upstream. * access_log: added new access_log command operator ``%VIRTUAL_CLUSTER_NAME%`` to retrieve the matched Virtual Cluster name. * api: added support for *xds.type.v3.TypedStruct* in addition to the now-deprecated *udpa.type.v1.TypedStruct* proto message, which is a wrapper proto used to encode typed JSON data in a *google.protobuf.Any* field. * aws_request_signing_filter: added :ref:`match_excluded_headers ` to the signing filter to optionally exclude request headers from signing. diff --git a/source/common/formatter/substitution_formatter.cc b/source/common/formatter/substitution_formatter.cc index cb78953df3650..42a790b9d9c32 100644 --- a/source/common/formatter/substitution_formatter.cc +++ b/source/common/formatter/substitution_formatter.cc @@ -818,6 +818,11 @@ StreamInfoFormatter::StreamInfoFormatter(const std::string& field_name) { StreamInfoAddressFieldExtractor::withPort([](const StreamInfo::StreamInfo& stream_info) { return stream_info.upstreamLocalAddress(); }); + } else if (field_name == "UPSTREAM_REQUEST_ATTEMPT_COUNT") { + field_extractor_ = std::make_unique( + [](const StreamInfo::StreamInfo& stream_info) { + return stream_info.attemptCount().value_or(0); + }); } else if (field_name == "DOWNSTREAM_LOCAL_ADDRESS") { field_extractor_ = StreamInfoAddressFieldExtractor::withPort([](const StreamInfo::StreamInfo& stream_info) { diff --git a/test/common/formatter/substitution_formatter_test.cc b/test/common/formatter/substitution_formatter_test.cc index 8cb401f900fcc..b732fec40c261 100644 --- a/test/common/formatter/substitution_formatter_test.cc +++ b/test/common/formatter/substitution_formatter_test.cc @@ -361,6 +361,28 @@ TEST(SubstitutionFormatterTest, streamInfoFormatter) { ProtoEq(ValueUtil::numberValue(1.0))); } + { + StreamInfoFormatter attempt_count_format("UPSTREAM_REQUEST_ATTEMPT_COUNT"); + absl::optional attempt_count{3}; + EXPECT_CALL(stream_info, attemptCount()).WillRepeatedly(Return(attempt_count)); + EXPECT_EQ("3", attempt_count_format.format(request_headers, response_headers, response_trailers, + stream_info, body)); + EXPECT_THAT(attempt_count_format.formatValue(request_headers, response_headers, + response_trailers, stream_info, body), + ProtoEq(ValueUtil::numberValue(3.0))); + } + + { + StreamInfoFormatter attempt_count_format("UPSTREAM_REQUEST_ATTEMPT_COUNT"); + absl::optional attempt_count; + EXPECT_CALL(stream_info, attemptCount()).WillRepeatedly(Return(attempt_count)); + EXPECT_EQ("0", attempt_count_format.format(request_headers, response_headers, response_trailers, + stream_info, body)); + EXPECT_THAT(attempt_count_format.formatValue(request_headers, response_headers, + response_trailers, stream_info, body), + ProtoEq(ValueUtil::numberValue(0.0))); + } + { StreamInfo::BytesMeterSharedPtr upstream_bytes_meter{ std::make_shared()}; From 8c9df52bb0b751d5a2c490b535f33fd6d8bedf80 Mon Sep 17 00:00:00 2001 From: Greg Greenway Date: Wed, 1 Dec 2021 13:34:14 -0800 Subject: [PATCH 42/51] docs: Add 1.20.1 release notes to main (#19163) Signed-off-by: Greg Greenway --- docs/root/version_history/current.rst | 3 -- docs/root/version_history/v1.20.1.rst | 32 +++++++++++++++++++ docs/root/version_history/version_history.rst | 1 + 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 docs/root/version_history/v1.20.1.rst diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 76c77a7e55d36..3caf3e6be8ebe 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -13,7 +13,6 @@ Minor Behavior Changes * bandwidth_limit: added :ref:`response trailers ` when request or response delay are enforced. * bandwidth_limit: added :ref:`bandwidth limit stats ` *request_enforced* and *response_enforced*. -* config: the log message for "gRPC config stream closed" now uses the most recent error message, and reports seconds instead of milliseconds for how long the most recent status has been received. * dns: now respecting the returned DNS TTL for resolved hosts, rather than always relying on the hard-coded :ref:`dns_refresh_rate. ` This behavior can be temporarily reverted by setting the runtime guard ``envoy.reloadable_features.use_dns_ttl`` to false. * http: envoy will now proxy 102 and 103 headers from upstream, though as with 100s only the first 1xx response headers will be sent. This behavioral change by can temporarily reverted by setting runtime guard ``envoy.reloadable_features.proxy_102_103`` to false. * http: usage of the experimental matching API is no longer guarded behind a feature flag, as the corresponding protobuf fields have been marked as WIP. @@ -28,9 +27,7 @@ Bug Fixes * ext_authz: fix the ext_authz http filter to correctly set response flags to ``UAEX`` when a connection is denied. * ext_authz: fix the ext_authz network filter to correctly set response flag and code details to ``UAEX`` when a connection is denied. -* listener: fixed the crash when updating listeners that do not bind to port. * listener: fixed issue where more than one listener could listen on the same port if using reuse port, thus randomly accepting connections on different listeners. This configuration is now rejected. -* tcp: fixed a bug where upstream circuit breakers applied HTTP per-request bounds to TCP connections. * thrift_proxy: fix the thrift_proxy connection manager to correctly report success/error response metrics when performing :ref:`payload passthrough `. Removed Config or Runtime diff --git a/docs/root/version_history/v1.20.1.rst b/docs/root/version_history/v1.20.1.rst new file mode 100644 index 0000000000000..5ee3ba7bc0c2c --- /dev/null +++ b/docs/root/version_history/v1.20.1.rst @@ -0,0 +1,32 @@ +1.20.1 (November 30, 2021) +========================== + +Incompatible Behavior Changes +----------------------------- +*Changes that are expected to cause an incompatibility if applicable; deployment changes are likely required* + +Minor Behavior Changes +---------------------- +*Changes that may cause incompatibilities for some users, but should not for most* + +* config: the log message for "gRPC config stream closed" now uses the most recent error message, and reports seconds instead of milliseconds for how long the most recent status has been received. + +Bug Fixes +--------- +*Changes expected to improve the state of the world and are unlikely to have negative effects* + +* http: remove redundant Warn log in HTTP codec. +* listener: fix a crash when updating any listener that does not bind to port. +* listener: listener add can reuse the listener socket of a draining filter chain listener and fix the request lost. +* mac: fix crash on startup on macOS 12 by changing the default allocator. +* tcp: fixed a bug where upstream circuit breakers applied HTTP per-request bounds to TCP connections. + +Removed Config or Runtime +------------------------- +*Normally occurs at the end of the* :ref:`deprecation period ` + +New Features +------------ + +Deprecated +---------- diff --git a/docs/root/version_history/version_history.rst b/docs/root/version_history/version_history.rst index 7cca61970b5bd..006173c8ade0e 100644 --- a/docs/root/version_history/version_history.rst +++ b/docs/root/version_history/version_history.rst @@ -7,6 +7,7 @@ Version history :titlesonly: current + v1.20.1 v1.20.0 v1.19.1 v1.19.0 From 77ca6cc0d9aaf0892aec3e2025fe2ad7cf0c39ff Mon Sep 17 00:00:00 2001 From: Douglas Reid Date: Wed, 1 Dec 2021 20:41:51 -0800 Subject: [PATCH 43/51] accesslogs: add CEL-based extension filter (#18363) This PR establishes the ability to filter access log production via CEL expressions over the set of Envoy attributes. This can simply the creation of Envoy access log filters, allowing complex tailoring. Risk Level: low Testing: unit Docs Changes: included Release Notes: updated Signed-off-by: Douglas Reid --- CODEOWNERS | 1 + api/BUILD | 1 + api/envoy/config/accesslog/v3/accesslog.proto | 1 + .../access_loggers/filters/cel/v3/BUILD | 9 +++ .../access_loggers/filters/cel/v3/cel.proto | 26 ++++++++ api/versioning/BUILD | 1 + bazel/repository_locations.bzl | 4 ++ .../api-v3/config/accesslog/accesslog.rst | 1 + docs/root/api-v3/config/accesslog/filters.rst | 8 +++ .../config/accesslog/filters/filters.rst | 8 +++ docs/root/api-v3/config/config.rst | 1 + docs/root/version_history/current.rst | 2 + .../access_loggers/filters/cel/BUILD | 63 ++++++++++++++++++ .../access_loggers/filters/cel/cel.cc | 35 ++++++++++ .../access_loggers/filters/cel/cel.h | 36 ++++++++++ .../access_loggers/filters/cel/config.cc | 65 ++++++++++++++++++ .../access_loggers/filters/cel/config.h | 36 ++++++++++ source/extensions/extensions_build_config.bzl | 1 + source/extensions/extensions_metadata.yaml | 5 ++ test/common/access_log/BUILD | 7 ++ .../common/access_log/access_log_impl_test.cc | 66 +++++++++++++++++++ tools/extensions/extensions_check.py | 2 +- 22 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 api/envoy/extensions/access_loggers/filters/cel/v3/BUILD create mode 100644 api/envoy/extensions/access_loggers/filters/cel/v3/cel.proto create mode 100644 docs/root/api-v3/config/accesslog/filters.rst create mode 100644 docs/root/api-v3/config/accesslog/filters/filters.rst create mode 100644 source/extensions/access_loggers/filters/cel/BUILD create mode 100644 source/extensions/access_loggers/filters/cel/cel.cc create mode 100644 source/extensions/access_loggers/filters/cel/cel.h create mode 100644 source/extensions/access_loggers/filters/cel/config.cc create mode 100644 source/extensions/access_loggers/filters/cel/config.h diff --git a/CODEOWNERS b/CODEOWNERS index 32cf634d3f16a..c8c3fa0e50a02 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -6,6 +6,7 @@ /api/ @envoyproxy/api-shepherds # access loggers /*/extensions/access_loggers/common @auni53 @zuercher +/*/extensions/access_loggers/filters/cel @dio @douglas-reid /*/extensions/access_loggers/open_telemetry @itamarkam @yanavlasov /*/extensions/access_loggers/stream @mattklein123 @davinci26 # compression extensions diff --git a/api/BUILD b/api/BUILD index ddd1f98b36cb7..b6438a01d7357 100644 --- a/api/BUILD +++ b/api/BUILD @@ -112,6 +112,7 @@ proto_library( "//envoy/data/dns/v3:pkg", "//envoy/data/tap/v3:pkg", "//envoy/extensions/access_loggers/file/v3:pkg", + "//envoy/extensions/access_loggers/filters/cel/v3:pkg", "//envoy/extensions/access_loggers/grpc/v3:pkg", "//envoy/extensions/access_loggers/open_telemetry/v3:pkg", "//envoy/extensions/access_loggers/stream/v3:pkg", diff --git a/api/envoy/config/accesslog/v3/accesslog.proto b/api/envoy/config/accesslog/v3/accesslog.proto index bb53286380c98..a89a4a709be1c 100644 --- a/api/envoy/config/accesslog/v3/accesslog.proto +++ b/api/envoy/config/accesslog/v3/accesslog.proto @@ -83,6 +83,7 @@ message AccessLogFilter { GrpcStatusFilter grpc_status_filter = 10; // Extension filter. + // [#extension-category: envoy.access_loggers.extension_filters] ExtensionFilter extension_filter = 11; // Metadata Filter diff --git a/api/envoy/extensions/access_loggers/filters/cel/v3/BUILD b/api/envoy/extensions/access_loggers/filters/cel/v3/BUILD new file mode 100644 index 0000000000000..ee92fb652582e --- /dev/null +++ b/api/envoy/extensions/access_loggers/filters/cel/v3/BUILD @@ -0,0 +1,9 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], +) diff --git a/api/envoy/extensions/access_loggers/filters/cel/v3/cel.proto b/api/envoy/extensions/access_loggers/filters/cel/v3/cel.proto new file mode 100644 index 0000000000000..8cb4d8b779259 --- /dev/null +++ b/api/envoy/extensions/access_loggers/filters/cel/v3/cel.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package envoy.extensions.access_loggers.filters.cel.v3; + +import "udpa/annotations/status.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.access_loggers.filters.cel.v3"; +option java_outer_classname = "CelProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: ExpressionFilter] +// [#extension: envoy.access_loggers.extension_filters.cel] + +// ExpressionFilter is an access logging filter that evaluates configured +// symbolic Common Expression Language expressions to inform the decision +// to generate an access log. +message ExpressionFilter { + // Expression that, when evaluated, will be used to filter access logs. + // Expressions are based on the set of Envoy :ref:`attributes `. + // The provided expression must evaluate to true for logging (expression errors are considered false). + // Examples: + // - `response.code >= 400` + // - `(connection.mtls && request.headers['x-log-mtls'] == 'true') || request.url_path.contains('v1beta3')` + string expression = 1; +} diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 24195d8d680dd..8be3045e4b9c6 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -49,6 +49,7 @@ proto_library( "//envoy/data/dns/v3:pkg", "//envoy/data/tap/v3:pkg", "//envoy/extensions/access_loggers/file/v3:pkg", + "//envoy/extensions/access_loggers/filters/cel/v3:pkg", "//envoy/extensions/access_loggers/grpc/v3:pkg", "//envoy/extensions/access_loggers/open_telemetry/v3:pkg", "//envoy/extensions/access_loggers/stream/v3:pkg", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 55e87d55300e2..ea3658f978710 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -859,6 +859,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/google/cel-cpp/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = [ + "envoy.access_loggers.extension_filters.cel", "envoy.access_loggers.wasm", "envoy.bootstrap.wasm", "envoy.rate_limit_descriptors.expr", @@ -882,6 +883,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/google/flatbuffers/archive/v{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = [ + "envoy.access_loggers.extension_filters.cel", "envoy.access_loggers.wasm", "envoy.bootstrap.wasm", "envoy.rate_limit_descriptors.expr", @@ -1078,6 +1080,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( # ANTLR has a runtime component, so is not purely build. use_category = ["dataplane_ext"], extensions = [ + "envoy.access_loggers.extension_filters.cel", "envoy.access_loggers.wasm", "envoy.bootstrap.wasm", "envoy.rate_limit_descriptors.expr", @@ -1098,6 +1101,7 @@ REPOSITORY_LOCATIONS_SPEC = dict( urls = ["https://github.com/antlr/antlr4/archive/{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = [ + "envoy.access_loggers.extension_filters.cel", "envoy.access_loggers.wasm", "envoy.bootstrap.wasm", "envoy.rate_limit_descriptors.expr", diff --git a/docs/root/api-v3/config/accesslog/accesslog.rst b/docs/root/api-v3/config/accesslog/accesslog.rst index dae49773f7881..6f265939d7eee 100644 --- a/docs/root/api-v3/config/accesslog/accesslog.rst +++ b/docs/root/api-v3/config/accesslog/accesslog.rst @@ -9,3 +9,4 @@ Access loggers v3/* ../../extensions/access_loggers/*/v3/* + ../../extensions/access_loggers/filters/*/v3/* diff --git a/docs/root/api-v3/config/accesslog/filters.rst b/docs/root/api-v3/config/accesslog/filters.rst new file mode 100644 index 0000000000000..692d0a18586b3 --- /dev/null +++ b/docs/root/api-v3/config/accesslog/filters.rst @@ -0,0 +1,8 @@ +Extension Filters +================= + +.. toctree:: + :glob: + :maxdepth: 2 + + filters/filters diff --git a/docs/root/api-v3/config/accesslog/filters/filters.rst b/docs/root/api-v3/config/accesslog/filters/filters.rst new file mode 100644 index 0000000000000..a27e04e376921 --- /dev/null +++ b/docs/root/api-v3/config/accesslog/filters/filters.rst @@ -0,0 +1,8 @@ +Extension Filters +================= + +.. toctree:: + :glob: + :maxdepth: 2 + + ../../../extensions/access_loggers/filters/*/v3/* diff --git a/docs/root/api-v3/config/config.rst b/docs/root/api-v3/config/config.rst index 6d4034ff5b8d8..605afc346a32d 100644 --- a/docs/root/api-v3/config/config.rst +++ b/docs/root/api-v3/config/config.rst @@ -9,6 +9,7 @@ Extensions filter/filter accesslog/accesslog + accesslog/filters rbac/rbac health_checker/health_checker transport_socket/transport_socket diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 3caf3e6be8ebe..8a6bd863c3ef8 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -54,6 +54,8 @@ Removed Config or Runtime New Features ------------ * access log: added :ref:`grpc_stream_retry_policy ` to the gRPC logger to reconnect when a connection fails to be established. +* access_log: added :ref:`METADATA` token to handle all types of metadata (DYNAMIC, CLUSTER, ROUTE). +* access_log: added a CEL extension filter to enable filtering of access logs based on Envoy attribute expressions. * access_log: added new access_log command operator ``%UPSTREAM_REQUEST_ATTEMPT_COUNT%`` to retrieve the number of times given request got attempted upstream. * access_log: added new access_log command operator ``%VIRTUAL_CLUSTER_NAME%`` to retrieve the matched Virtual Cluster name. * api: added support for *xds.type.v3.TypedStruct* in addition to the now-deprecated *udpa.type.v1.TypedStruct* proto message, which is a wrapper proto used to encode typed JSON data in a *google.protobuf.Any* field. diff --git a/source/extensions/access_loggers/filters/cel/BUILD b/source/extensions/access_loggers/filters/cel/BUILD new file mode 100644 index 0000000000000..4e53f30780d7a --- /dev/null +++ b/source/extensions/access_loggers/filters/cel/BUILD @@ -0,0 +1,63 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "cel_lib", + srcs = ["cel.cc"], + hdrs = ["cel.h"], + extra_visibility = [ + "//test:__subpackages__", + ], + deps = [ + "//envoy/access_log:access_log_interface", + "//envoy/http:header_map_interface", + "//envoy/stream_info:stream_info_interface", + "//source/common/access_log:access_log_lib", + "//source/common/config:utility_lib", + "//source/common/protobuf", + "//source/common/protobuf:utility_lib", + "//source/extensions/filters/common/expr:evaluator_lib", + ], +) + +envoy_cc_extension( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + copts = select({ + "//bazel:windows_x86_64": [], # TODO: fix the windows ANTLR build + "//conditions:default": [ + "-DUSE_CEL_PARSER", + ], + }), + extra_visibility = [ + "//test:__subpackages__", + ], + deps = [ + ":cel_lib", + "//envoy/access_log:access_log_interface", + "//envoy/http:header_map_interface", + "//envoy/registry", + "//envoy/stream_info:stream_info_interface", + "//source/common/access_log:access_log_lib", + "//source/common/config:utility_lib", + "//source/common/protobuf", + "//source/common/protobuf:utility_lib", + "//source/extensions/filters/common/expr:evaluator_lib", + "@envoy_api//envoy/extensions/access_loggers/filters/cel/v3:pkg_cc_proto", + ] + select( + { + "//bazel:windows_x86_64": [], + "//conditions:default": [ + "@com_google_cel_cpp//parser", + ], + }, + ), +) diff --git a/source/extensions/access_loggers/filters/cel/cel.cc b/source/extensions/access_loggers/filters/cel/cel.cc new file mode 100644 index 0000000000000..a9c87da2e1a01 --- /dev/null +++ b/source/extensions/access_loggers/filters/cel/cel.cc @@ -0,0 +1,35 @@ +#include "source/extensions/access_loggers/filters/cel/cel.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace Filters { +namespace CEL { + +namespace Expr = Envoy::Extensions::Filters::Common::Expr; + +CELAccessLogExtensionFilter::CELAccessLogExtensionFilter( + Expr::Builder& builder, const google::api::expr::v1alpha1::Expr& input_expr) + : parsed_expr_(input_expr) { + compiled_expr_ = Expr::createExpression(builder, parsed_expr_); +} + +bool CELAccessLogExtensionFilter::evaluate( + const StreamInfo::StreamInfo& stream_info, const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers) const { + Protobuf::Arena arena; + auto eval_status = Expr::evaluate(*compiled_expr_, arena, stream_info, &request_headers, + &response_headers, &response_trailers); + if (!eval_status.has_value() || eval_status.value().IsError()) { + return false; + } + auto result = eval_status.value(); + return result.IsBool() ? result.BoolOrDie() : false; +} + +} // namespace CEL +} // namespace Filters +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/filters/cel/cel.h b/source/extensions/access_loggers/filters/cel/cel.h new file mode 100644 index 0000000000000..911f04106b60f --- /dev/null +++ b/source/extensions/access_loggers/filters/cel/cel.h @@ -0,0 +1,36 @@ +#include "envoy/access_log/access_log.h" +#include "envoy/http/header_map.h" +#include "envoy/stream_info/stream_info.h" + +#include "source/common/access_log/access_log_impl.h" +#include "source/common/config/utility.h" +#include "source/common/protobuf/message_validator_impl.h" +#include "source/common/protobuf/protobuf.h" +#include "source/common/protobuf/utility.h" +#include "source/extensions/filters/common/expr/evaluator.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace Filters { +namespace CEL { + +class CELAccessLogExtensionFilter : public AccessLog::Filter { +public: + CELAccessLogExtensionFilter(Extensions::Filters::Common::Expr::Builder&, + const google::api::expr::v1alpha1::Expr&); + + bool evaluate(const StreamInfo::StreamInfo& info, const Http::RequestHeaderMap& request_headers, + const Http::ResponseHeaderMap& response_headers, + const Http::ResponseTrailerMap& response_trailers) const override; + +private: + const google::api::expr::v1alpha1::Expr parsed_expr_; + Extensions::Filters::Common::Expr::ExpressionPtr compiled_expr_; +}; + +} // namespace CEL +} // namespace Filters +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/filters/cel/config.cc b/source/extensions/access_loggers/filters/cel/config.cc new file mode 100644 index 0000000000000..0df41fc19c525 --- /dev/null +++ b/source/extensions/access_loggers/filters/cel/config.cc @@ -0,0 +1,65 @@ +#include "source/extensions/access_loggers/filters/cel/config.h" + +#include "envoy/extensions/access_loggers/filters/cel/v3/cel.pb.h" + +#include "source/extensions/access_loggers/filters/cel/cel.h" + +#if defined(USE_CEL_PARSER) +#include "parser/parser.h" +#endif + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace Filters { +namespace CEL { + +Envoy::AccessLog::FilterPtr CELAccessLogExtensionFilterFactory::createFilter( + const envoy::config::accesslog::v3::ExtensionFilter& config, Runtime::Loader&, + Random::RandomGenerator&) { + + // TODO(douglas-reid): use factory_context validation. likely needs update to + // createFilter signature to pass in validation visitor. + auto factory_config = Config::Utility::translateToFactoryConfig( + config, Envoy::ProtobufMessage::getNullValidationVisitor(), *this); + +#if defined(USE_CEL_PARSER) + envoy::extensions::access_loggers::filters::cel::v3::ExpressionFilter cel_config = + *dynamic_cast( + factory_config.get()); + + auto parse_status = google::api::expr::parser::Parse(cel_config.expression()); + if (!parse_status.ok()) { + throw EnvoyException("Not able to parse filter expression: " + + parse_status.status().ToString()); + } + + return std::make_unique(getOrCreateBuilder(), + parse_status.value().expr()); +#else + throw EnvoyException("CEL is not available for use in this environment."); +#endif +} + +ProtobufTypes::MessagePtr CELAccessLogExtensionFilterFactory::createEmptyConfigProto() { + return std::make_unique(); +} + +Extensions::Filters::Common::Expr::Builder& +CELAccessLogExtensionFilterFactory::getOrCreateBuilder() { + if (expr_builder_ == nullptr) { + expr_builder_ = Extensions::Filters::Common::Expr::createBuilder(nullptr); + } + return *expr_builder_; +} + +/** + * Static registration for the CELAccessLogExtensionFilter. @see RegisterFactory. + */ +REGISTER_FACTORY(CELAccessLogExtensionFilterFactory, Envoy::AccessLog::ExtensionFilterFactory); + +} // namespace CEL +} // namespace Filters +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/filters/cel/config.h b/source/extensions/access_loggers/filters/cel/config.h new file mode 100644 index 0000000000000..0266af6fbb0ab --- /dev/null +++ b/source/extensions/access_loggers/filters/cel/config.h @@ -0,0 +1,36 @@ +#include "envoy/access_log/access_log.h" +#include "envoy/http/header_map.h" +#include "envoy/registry/registry.h" +#include "envoy/stream_info/stream_info.h" + +#include "source/common/access_log/access_log_impl.h" +#include "source/common/config/utility.h" +#include "source/common/protobuf/message_validator_impl.h" +#include "source/common/protobuf/protobuf.h" +#include "source/common/protobuf/utility.h" +#include "source/extensions/filters/common/expr/evaluator.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace Filters { +namespace CEL { + +class CELAccessLogExtensionFilterFactory : public Envoy::AccessLog::ExtensionFilterFactory { +public: + Envoy::AccessLog::FilterPtr + createFilter(const envoy::config::accesslog::v3::ExtensionFilter& config, Runtime::Loader&, + Random::RandomGenerator&) override; + ProtobufTypes::MessagePtr createEmptyConfigProto() override; + std::string name() const override { return "envoy.access_loggers.extension_filters.cel"; } + +private: + Extensions::Filters::Common::Expr::Builder& getOrCreateBuilder(); + Extensions::Filters::Common::Expr::BuilderPtr expr_builder_; +}; + +} // namespace CEL +} // namespace Filters +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index cd6a62d3d5c42..af6c536084391 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -5,6 +5,7 @@ EXTENSIONS = { # "envoy.access_loggers.file": "//source/extensions/access_loggers/file:config", + "envoy.access_loggers.extension_filters.cel": "//source/extensions/access_loggers/filters/cel:config", "envoy.access_loggers.http_grpc": "//source/extensions/access_loggers/grpc:http_config", "envoy.access_loggers.tcp_grpc": "//source/extensions/access_loggers/grpc:tcp_config", "envoy.access_loggers.open_telemetry": "//source/extensions/access_loggers/open_telemetry:config", diff --git a/source/extensions/extensions_metadata.yaml b/source/extensions/extensions_metadata.yaml index 6a4a403220fdb..3cf47195acf25 100644 --- a/source/extensions/extensions_metadata.yaml +++ b/source/extensions/extensions_metadata.yaml @@ -3,6 +3,11 @@ envoy.access_loggers.file: - envoy.access_loggers security_posture: robust_to_untrusted_downstream status: stable +envoy.access_loggers.extension_filters.cel: + categories: + - envoy.access_loggers.extension_filters + security_posture: unknown + status: alpha envoy.access_loggers.http_grpc: categories: - envoy.access_loggers diff --git a/test/common/access_log/BUILD b/test/common/access_log/BUILD index 5a1908dfb08ad..bc8eb168dccd8 100644 --- a/test/common/access_log/BUILD +++ b/test/common/access_log/BUILD @@ -11,10 +11,17 @@ envoy_package() envoy_cc_test( name = "access_log_impl_test", srcs = ["access_log_impl_test.cc"], + copts = select({ + "//bazel:windows_x86_64": [], # TODO: fix the windows ANTLR build + "//conditions:default": [ + "-DUSE_CEL_PARSER", + ], + }), deps = [ "//source/common/access_log:access_log_lib", "//source/common/stream_info:utility_lib", "//source/extensions/access_loggers/file:config", + "//source/extensions/access_loggers/filters/cel:config", "//source/extensions/access_loggers/grpc:http_config", "//source/extensions/access_loggers/grpc:tcp_config", "//source/extensions/access_loggers/stream:config", diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index d2d4c291f110d..c5b24903996df 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -1598,6 +1598,72 @@ name: accesslog } } +#if defined(USE_CEL_PARSER) +TEST_F(AccessLogImplTest, CelExtensionFilter) { + const std::string yaml = R"EOF( +name: accesslog +filter: + extension_filter: + name: cel_extension_filter + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: "(request.headers['log'] == 'true') && (response.code >= 400)" +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + )EOF"; + + InstanceSharedPtr logger = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); + + request_headers_.addCopy("log", "true"); + stream_info_.response_code_ = 404; + EXPECT_CALL(*file_, write(_)); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + + request_headers_.remove("log"); + EXPECT_CALL(*file_, write(_)).Times(0); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); +} + +TEST_F(AccessLogImplTest, CelExtensionFilterExpressionError) { + const std::string yaml = R"EOF( +name: accesslog +filter: + extension_filter: + name: cel_extension_filter + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: "foo['test']" +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + )EOF"; + + InstanceSharedPtr logger = AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); + + EXPECT_CALL(*file_, write(_)).Times(0); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); +} + +TEST_F(AccessLogImplTest, CelExtensionFilterExpressionUnparsable) { + const std::string yaml = R"EOF( +name: accesslog +filter: + extension_filter: + name: cel_extension_filter + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.filters.cel.v3.ExpressionFilter + expression: "(+++" +typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + )EOF"; + + EXPECT_THROW_WITH_REGEX(AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_), + EnvoyException, "Not able to parse filter expression: .*"); +} +#endif // USE_CEL_PARSER + // Test that the deprecated extension names are disabled by default. // TODO(zuercher): remove when envoy.deprecated_features.allow_deprecated_extension_names is removed TEST_F(AccessLogImplTest, DEPRECATED_FEATURE_TEST(DeprecatedExtensionFilterName)) { diff --git a/tools/extensions/extensions_check.py b/tools/extensions/extensions_check.py index c9c204050f38d..313206572d122 100644 --- a/tools/extensions/extensions_check.py +++ b/tools/extensions/extensions_check.py @@ -55,7 +55,7 @@ "envoy.stats_sinks", "envoy.thrift_proxy.filters", "envoy.tracers", "envoy.sip_proxy.filters", "envoy.transport_sockets.downstream", "envoy.transport_sockets.upstream", "envoy.tls.cert_validator", "envoy.upstreams", "envoy.wasm.runtime", "envoy.common.key_value", - "envoy.network.dns_resolver", "envoy.rbac.matchers") + "envoy.network.dns_resolver", "envoy.rbac.matchers", "envoy.access_loggers.extension_filters") EXTENSION_STATUS_VALUES = ( # This extension is stable and is expected to be production usable. From 4b5eee6b8ec3fc84fd1f7e6ee684947724843e47 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Thu, 2 Dec 2021 16:25:10 +0100 Subject: [PATCH 44/51] Support proactive checks in overload manager Api (#18256) Signed-off-by: Kateryna Nezdolii --- envoy/server/BUILD | 9 ++ envoy/server/overload/BUILD | 1 + .../overload/thread_local_overload_state.h | 46 +++++++ envoy/server/proactive_resource_monitor.h | 86 ++++++++++++ envoy/server/resource_monitor.h | 42 +++--- envoy/server/resource_monitor_config.h | 21 +++ .../fixed_heap/fixed_heap_monitor.cc | 2 +- .../fixed_heap/fixed_heap_monitor.h | 2 +- .../injected_resource_monitor.cc | 2 +- .../injected_resource_monitor.h | 2 +- source/server/admin/admin.h | 3 + source/server/overload_manager_impl.cc | 100 ++++++++++++-- source/server/overload_manager_impl.h | 7 +- .../fixed_heap/fixed_heap_monitor_test.cc | 2 +- .../injected_resource_monitor_test.cc | 2 +- test/integration/fake_resource_monitor.cc | 2 +- test/integration/fake_resource_monitor.h | 2 +- test/mocks/server/overload_manager.cc | 4 + test/mocks/server/overload_manager.h | 3 + test/server/overload_manager_impl_test.cc | 124 +++++++++++++++++- 20 files changed, 413 insertions(+), 49 deletions(-) create mode 100644 envoy/server/proactive_resource_monitor.h diff --git a/envoy/server/BUILD b/envoy/server/BUILD index ea94574d4b41f..bd34c1e8e5fdf 100644 --- a/envoy/server/BUILD +++ b/envoy/server/BUILD @@ -294,6 +294,14 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "proactive_resource_monitor_interface", + hdrs = ["proactive_resource_monitor.h"], + deps = [ + "//envoy/stats:stats_interface", + ], +) + envoy_cc_library( name = "request_id_extension_config_interface", hdrs = ["request_id_extension_config.h"], @@ -309,6 +317,7 @@ envoy_cc_library( hdrs = ["resource_monitor_config.h"], deps = [ ":options_interface", + ":proactive_resource_monitor_interface", ":resource_monitor_interface", "//envoy/api:api_interface", "//envoy/event:dispatcher_interface", diff --git a/envoy/server/overload/BUILD b/envoy/server/overload/BUILD index 6d937fb8d5e18..fd1bf3e0bc7be 100644 --- a/envoy/server/overload/BUILD +++ b/envoy/server/overload/BUILD @@ -25,6 +25,7 @@ envoy_cc_library( deps = [ "//envoy/event:scaled_range_timer_manager_interface", "//envoy/event:timer_interface", + "//envoy/server:proactive_resource_monitor_interface", "//envoy/thread_local:thread_local_object", "//source/common/common:interval_value", ], diff --git a/envoy/server/overload/thread_local_overload_state.h b/envoy/server/overload/thread_local_overload_state.h index 9a57400d8eb31..35f36050945e4 100644 --- a/envoy/server/overload/thread_local_overload_state.h +++ b/envoy/server/overload/thread_local_overload_state.h @@ -5,13 +5,33 @@ #include "envoy/common/pure.h" #include "envoy/event/scaled_range_timer_manager.h" #include "envoy/event/timer.h" +#include "envoy/server/proactive_resource_monitor.h" #include "envoy/thread_local/thread_local_object.h" #include "source/common/common/interval_value.h" +#include "source/common/singleton/const_singleton.h" namespace Envoy { namespace Server { +enum class OverloadProactiveResourceName { + GlobalDownstreamMaxConnections, +}; + +class OverloadProactiveResourceNameValues { +public: + // Overload action to stop accepting new HTTP requests. + const std::string GlobalDownstreamMaxConnections = + "envoy.resource_monitors.global_downstream_max_connections"; + + absl::flat_hash_map + proactive_action_name_to_resource_ = { + {GlobalDownstreamMaxConnections, + OverloadProactiveResourceName::GlobalDownstreamMaxConnections}}; +}; + +using OverloadProactiveResources = ConstSingleton; + /** * Tracks the state of an overload action. The state is a number between 0 and 1 that represents the * level of saturation. The values are categorized in two groups: @@ -46,6 +66,32 @@ class ThreadLocalOverloadState : public ThreadLocal::ThreadLocalObject { public: // Get a thread-local reference to the value for the given action key. virtual const OverloadActionState& getState(const std::string& action) PURE; + /** + * Invokes the corresponding resource monitor to allocate resource for given resource monitor in + * a thread safe manner. Returns true if there is enough resource quota available and allocation + * has succeeded, false if allocation failed or resource is not registered. + * @param name of corresponding resource monitor. + * @param increment to add to current resource usage value within monitor. + */ + virtual bool tryAllocateResource(OverloadProactiveResourceName resource_name, + int64_t increment) PURE; + /** + * Invokes the corresponding resource monitor to deallocate resource for given resource monitor in + * a thread safe manner. Returns true if there is enough resource quota available and deallocation + * has succeeded, false if deallocation failed or resource is not registered. + * @param name of corresponding resource monitor. + * @param decrement to subtract from current resource usage value within monitor. + */ + virtual bool tryDeallocateResource(OverloadProactiveResourceName resource_name, + int64_t decrement) PURE; + + /** + * TODO(nezdolik) remove this method once downstream connection tracking is fully moved to + * overload manager. Checks if resource monitor is registered and resource usage tracking is + * enabled in overload manager. Returns true if resource monitor is registered, false otherwise. + * @param name of resource monitor to check. + */ + virtual bool isResourceMonitorEnabled(OverloadProactiveResourceName resource_name) PURE; }; } // namespace Server diff --git a/envoy/server/proactive_resource_monitor.h b/envoy/server/proactive_resource_monitor.h new file mode 100644 index 0000000000000..ee37ccf094fd5 --- /dev/null +++ b/envoy/server/proactive_resource_monitor.h @@ -0,0 +1,86 @@ +#pragma once + +#include + +#include "envoy/common/pure.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats.h" + +#include "source/common/common/assert.h" +#include "source/common/stats/symbol_table_impl.h" + +namespace Envoy { +namespace Server { + +class ProactiveResourceMonitor { +public: + ProactiveResourceMonitor() = default; + virtual ~ProactiveResourceMonitor() = default; + /** + * Tries to allocate resource for given resource monitor in thread safe manner. + * Returns true if there is enough resource quota available and allocation has succeeded, false + * otherwise. + * @param increment to add to current resource usage value and compare against configured max + * threshold. + */ + virtual bool tryAllocateResource(int64_t increment) PURE; + /** + * Tries to deallocate resource for given resource monitor in thread safe manner. + * Returns true if there is enough resource quota available and deallocation has succeeded, false + * otherwise. + * @param decrement to subtract from current resource usage value. + */ + virtual bool tryDeallocateResource(int64_t decrement) PURE; + /** + * Returns current resource usage (most recent read) tracked by monitor. + */ + virtual int64_t currentResourceUsage() const PURE; + /** + * Returns max resource usage configured in monitor. + */ + virtual int64_t maxResourceUsage() const PURE; +}; + +using ProactiveResourceMonitorPtr = std::unique_ptr; + +class ProactiveResource { +public: + ProactiveResource(const std::string& name, ProactiveResourceMonitorPtr monitor, + Stats::Scope& stats_scope) + : name_(name), monitor_(std::move(monitor)), + failed_updates_counter_(makeCounter(stats_scope, name, "failed_updates")) {} + + bool tryAllocateResource(int64_t increment) { + if (monitor_->tryAllocateResource(increment)) { + return true; + } else { + failed_updates_counter_.inc(); + return false; + } + } + + bool tryDeallocateResource(int64_t decrement) { + if (monitor_->tryDeallocateResource(decrement)) { + return true; + } else { + failed_updates_counter_.inc(); + return false; + } + } + + int64_t currentResourceUsage() { return monitor_->currentResourceUsage(); } + +private: + const std::string name_; + ProactiveResourceMonitorPtr monitor_; + Stats::Counter& failed_updates_counter_; + + Stats::Counter& makeCounter(Stats::Scope& scope, absl::string_view a, absl::string_view b) { + Stats::StatNameManagedStorage stat_name(absl::StrCat("overload.", a, ".", b), + scope.symbolTable()); + return scope.counterFromStatName(stat_name.statName()); + } +}; + +} // namespace Server +} // namespace Envoy diff --git a/envoy/server/resource_monitor.h b/envoy/server/resource_monitor.h index 4eb947527ade6..e2b0d0647572f 100644 --- a/envoy/server/resource_monitor.h +++ b/envoy/server/resource_monitor.h @@ -5,6 +5,8 @@ #include "envoy/common/exception.h" #include "envoy/common/pure.h" +#include "source/common/common/assert.h" + namespace Envoy { namespace Server { @@ -18,36 +20,36 @@ struct ResourceUsage { double resource_pressure_; }; -class ResourceMonitor { +/** + * Notifies caller of updated resource usage. + */ +class ResourceUpdateCallbacks { public: - virtual ~ResourceMonitor() = default; + virtual ~ResourceUpdateCallbacks() = default; + + /** + * Called when the request for updated resource usage succeeds. + * @param usage the updated resource usage + */ + virtual void onSuccess(const ResourceUsage& usage) PURE; /** - * Notifies caller of updated resource usage. + * Called when the request for updated resource usage fails. + * @param error the exception caught when trying to get updated resource usage */ - class Callbacks { - public: - virtual ~Callbacks() = default; - - /** - * Called when the request for updated resource usage succeeds. - * @param usage the updated resource usage - */ - virtual void onSuccess(const ResourceUsage& usage) PURE; - - /** - * Called when the request for updated resource usage fails. - * @param error the exception caught when trying to get updated resource usage - */ - virtual void onFailure(const EnvoyException& error) PURE; - }; + virtual void onFailure(const EnvoyException& error) PURE; +}; + +class ResourceMonitor { +public: + virtual ~ResourceMonitor() = default; /** * Recalculate resource usage. * This must be non-blocking so if RPCs need to be made they should be * done asynchronously and invoke the callback when finished. */ - virtual void updateResourceUsage(Callbacks& callbacks) PURE; + virtual void updateResourceUsage(ResourceUpdateCallbacks& callbacks) PURE; }; using ResourceMonitorPtr = std::unique_ptr; diff --git a/envoy/server/resource_monitor_config.h b/envoy/server/resource_monitor_config.h index 9f680f44f8f2a..a7d9086c28f0d 100644 --- a/envoy/server/resource_monitor_config.h +++ b/envoy/server/resource_monitor_config.h @@ -6,6 +6,7 @@ #include "envoy/event/dispatcher.h" #include "envoy/protobuf/message_validator.h" #include "envoy/server/options.h" +#include "envoy/server/proactive_resource_monitor.h" #include "envoy/server/resource_monitor.h" #include "source/common/protobuf/protobuf.h" @@ -64,6 +65,26 @@ class ResourceMonitorFactory : public Config::TypedFactory { std::string category() const override { return "envoy.resource_monitors"; } }; +class ProactiveResourceMonitorFactory : public Config::TypedFactory { +public: + ~ProactiveResourceMonitorFactory() override = default; + + /** + * Create a particular proactive resource monitor implementation. + * @param config const ProtoBuf::Message& supplies the config for the proactive resource monitor + * implementation. + * @param context ResourceMonitorFactoryContext& supplies the resource monitor's context. + * @return ProactiveResourceMonitorPtr the resource monitor instance. Should not be nullptr. + * @throw EnvoyException if the implementation is unable to produce an instance with + * the provided parameters. + */ + virtual ProactiveResourceMonitorPtr + createProactiveResourceMonitor(const Protobuf::Message& config, + ResourceMonitorFactoryContext& context) PURE; + + std::string category() const override { return "envoy.resource_monitors"; } +}; + } // namespace Configuration } // namespace Server } // namespace Envoy diff --git a/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc index 106c44620b68d..6060adab4c975 100644 --- a/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc +++ b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.cc @@ -21,7 +21,7 @@ FixedHeapMonitor::FixedHeapMonitor( ASSERT(max_heap_ > 0); } -void FixedHeapMonitor::updateResourceUsage(Server::ResourceMonitor::Callbacks& callbacks) { +void FixedHeapMonitor::updateResourceUsage(Server::ResourceUpdateCallbacks& callbacks) { const size_t physical = stats_->reservedHeapBytes(); const size_t unmapped = stats_->unmappedHeapBytes(); ASSERT(physical >= unmapped); diff --git a/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h index a54162ebb31f9..84ed32c80e797 100644 --- a/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h +++ b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h @@ -31,7 +31,7 @@ class FixedHeapMonitor : public Server::ResourceMonitor { const envoy::extensions::resource_monitors::fixed_heap::v3::FixedHeapConfig& config, std::unique_ptr stats = std::make_unique()); - void updateResourceUsage(Server::ResourceMonitor::Callbacks& callbacks) override; + void updateResourceUsage(Server::ResourceUpdateCallbacks& callbacks) override; private: const uint64_t max_heap_; diff --git a/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.cc b/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.cc index d6797ac85ec2f..a2799ac6ff15d 100644 --- a/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.cc +++ b/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.cc @@ -23,7 +23,7 @@ InjectedResourceMonitor::InjectedResourceMonitor( void InjectedResourceMonitor::onFileChanged() { file_changed_ = true; } -void InjectedResourceMonitor::updateResourceUsage(Server::ResourceMonitor::Callbacks& callbacks) { +void InjectedResourceMonitor::updateResourceUsage(Server::ResourceUpdateCallbacks& callbacks) { if (file_changed_) { file_changed_ = false; TRY_ASSERT_MAIN_THREAD { diff --git a/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.h b/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.h index 86210c112a076..59bf32375c562 100644 --- a/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.h +++ b/source/extensions/resource_monitors/injected_resource/injected_resource_monitor.h @@ -25,7 +25,7 @@ class InjectedResourceMonitor : public Server::ResourceMonitor { Server::Configuration::ResourceMonitorFactoryContext& context); // Server::ResourceMonitor - void updateResourceUsage(Server::ResourceMonitor::Callbacks& callbacks) override; + void updateResourceUsage(Server::ResourceUpdateCallbacks& callbacks) override; protected: virtual void onFileChanged(); diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index 7ee70f069d01f..e255c27d3367e 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -270,6 +270,9 @@ class AdminImpl : public Admin, struct NullThreadLocalOverloadState : public ThreadLocalOverloadState { NullThreadLocalOverloadState(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher) {} const OverloadActionState& getState(const std::string&) override { return inactive_; } + bool tryAllocateResource(OverloadProactiveResourceName, int64_t) override { return false; } + bool tryDeallocateResource(OverloadProactiveResourceName, int64_t) override { return false; } + bool isResourceMonitorEnabled(OverloadProactiveResourceName) override { return false; } Event::Dispatcher& dispatcher_; const OverloadActionState inactive_ = OverloadActionState::inactive(); }; diff --git a/source/server/overload_manager_impl.cc b/source/server/overload_manager_impl.cc index 1e999e718ce94..4a274c145e1ba 100644 --- a/source/server/overload_manager_impl.cc +++ b/source/server/overload_manager_impl.cc @@ -25,9 +25,13 @@ namespace Server { */ class ThreadLocalOverloadStateImpl : public ThreadLocalOverloadState { public: - explicit ThreadLocalOverloadStateImpl(const NamedOverloadActionSymbolTable& action_symbol_table) + explicit ThreadLocalOverloadStateImpl( + const NamedOverloadActionSymbolTable& action_symbol_table, + std::shared_ptr>& + proactive_resources) : action_symbol_table_(action_symbol_table), - actions_(action_symbol_table.size(), OverloadActionState(UnitFloat::min())) {} + actions_(action_symbol_table.size(), OverloadActionState(UnitFloat::min())), + proactive_resources_(proactive_resources) {} const OverloadActionState& getState(const std::string& action) override { if (const auto symbol = action_symbol_table_.lookup(action); symbol != absl::nullopt) { @@ -40,10 +44,44 @@ class ThreadLocalOverloadStateImpl : public ThreadLocalOverloadState { actions_[action.index()] = state; } + bool tryAllocateResource(OverloadProactiveResourceName resource_name, + int64_t increment) override { + const auto proactive_resource = proactive_resources_->find(resource_name); + if (proactive_resource != proactive_resources_->end()) { + return proactive_resource->second.tryAllocateResource(increment); + } else { + ENVOY_LOG_MISC(warn, " {Failed to allocate unknown proactive resource }"); + // Resource monitor is not configured. + return false; + } + } + + bool tryDeallocateResource(OverloadProactiveResourceName resource_name, + int64_t decrement) override { + const auto proactive_resource = proactive_resources_->find(resource_name); + if (proactive_resource != proactive_resources_->end()) { + if (proactive_resource->second.tryDeallocateResource(decrement)) { + return true; + } else { + return false; + } + } else { + ENVOY_LOG_MISC(warn, " {Failed to deallocate unknown proactive resource }"); + return false; + } + } + + bool isResourceMonitorEnabled(OverloadProactiveResourceName resource_name) override { + const auto proactive_resource = proactive_resources_->find(resource_name); + return proactive_resource != proactive_resources_->end(); + } + private: static const OverloadActionState always_inactive_; const NamedOverloadActionSymbolTable& action_symbol_table_; std::vector actions_; + std::shared_ptr> + proactive_resources_; }; const OverloadActionState ThreadLocalOverloadStateImpl::always_inactive_{UnitFloat::min()}; @@ -268,19 +306,47 @@ OverloadManagerImpl::OverloadManagerImpl(Event::Dispatcher& dispatcher, Stats::S Api::Api& api, const Server::Options& options) : started_(false), dispatcher_(dispatcher), tls_(slot_allocator), refresh_interval_( - std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(config, refresh_interval, 1000))) { + std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(config, refresh_interval, 1000))), + proactive_resources_( + std::make_unique< + absl::node_hash_map>()) { Configuration::ResourceMonitorFactoryContextImpl context(dispatcher, options, api, validation_visitor); + // We should hide impl details from users, for them there should be no distinction between + // proactive and regular resource monitors in configuration API. But internally we will maintain + // two distinct collections of proactive and regular resources. Proactive resources are not + // subject to periodic flushes and can be recalculated/updated on demand by invoking + // `tryAllocateResource/tryDeallocateResource` via thread local overload state. for (const auto& resource : config.resource_monitors()) { const auto& name = resource.name(); - ENVOY_LOG(debug, "Adding resource monitor for {}", name); - auto& factory = - Config::Utility::getAndCheckFactory(resource); - auto config = Config::Utility::translateToFactoryConfig(resource, validation_visitor, factory); - auto monitor = factory.createResourceMonitor(*config, context); - - auto result = resources_.try_emplace(name, name, std::move(monitor), *this, stats_scope); - if (!result.second) { + // Check if it is a proactive resource. + auto proactive_resource_it = + OverloadProactiveResources::get().proactive_action_name_to_resource_.find(name); + ENVOY_LOG(debug, "Evaluating resource {}", name); + bool result = false; + if (proactive_resource_it != + OverloadProactiveResources::get().proactive_action_name_to_resource_.end()) { + ENVOY_LOG(debug, "Adding proactive resource monitor for {}", name); + auto& factory = + Config::Utility::getAndCheckFactory( + resource); + auto config = + Config::Utility::translateToFactoryConfig(resource, validation_visitor, factory); + auto monitor = factory.createProactiveResourceMonitor(*config, context); + result = + proactive_resources_ + ->try_emplace(proactive_resource_it->second, name, std::move(monitor), stats_scope) + .second; + } else { + ENVOY_LOG(debug, "Adding resource monitor for {}", name); + auto& factory = + Config::Utility::getAndCheckFactory(resource); + auto config = + Config::Utility::translateToFactoryConfig(resource, validation_visitor, factory); + auto monitor = factory.createResourceMonitor(*config, context); + result = resources_.try_emplace(name, name, std::move(monitor), *this, stats_scope).second; + } + if (!result) { throw EnvoyException(absl::StrCat("Duplicate resource monitor ", name)); } } @@ -315,12 +381,15 @@ OverloadManagerImpl::OverloadManagerImpl(Event::Dispatcher& dispatcher, Stats::S for (const auto& trigger : action.triggers()) { const std::string& resource = trigger.name(); + auto proactive_resource_it = + OverloadProactiveResources::get().proactive_action_name_to_resource_.find(resource); - if (resources_.find(resource) == resources_.end()) { + if (resources_.find(resource) == resources_.end() && + proactive_resource_it == + OverloadProactiveResources::get().proactive_action_name_to_resource_.end()) { throw EnvoyException( fmt::format("Unknown trigger resource {} for overload action {}", resource, name)); } - resource_to_actions_.insert(std::make_pair(resource, symbol)); } } @@ -331,7 +400,8 @@ void OverloadManagerImpl::start() { started_ = true; tls_.set([this](Event::Dispatcher&) { - return std::make_shared(action_symbol_table_); + return std::make_shared(action_symbol_table_, + proactive_resources_); }); if (resources_.empty()) { @@ -364,6 +434,8 @@ void OverloadManagerImpl::stop() { // Clear the resource map to block on any pending updates. resources_.clear(); + + // TODO(nezdolik): wrap proactive monitors into atomic? and clear it here } bool OverloadManagerImpl::registerForAction(const std::string& action, diff --git a/source/server/overload_manager_impl.h b/source/server/overload_manager_impl.h index 784308b1f4b89..b7e4d3e12fb03 100644 --- a/source/server/overload_manager_impl.h +++ b/source/server/overload_manager_impl.h @@ -132,12 +132,12 @@ class OverloadManagerImpl : Logger::Loggable, public OverloadM private: using FlushEpochId = uint64_t; - class Resource : public ResourceMonitor::Callbacks { + class Resource : public ResourceUpdateCallbacks { public: Resource(const std::string& name, ResourceMonitorPtr monitor, OverloadManagerImpl& manager, Stats::Scope& stats_scope); - // ResourceMonitor::Callbacks + // ResourceMonitor::ResourceUpdateCallbacks void onSuccess(const ResourceUsage& usage) override; void onFailure(const EnvoyException& error) override; @@ -173,6 +173,9 @@ class OverloadManagerImpl : Logger::Loggable, public OverloadM const std::chrono::milliseconds refresh_interval_; Event::TimerPtr timer_; absl::node_hash_map resources_; + std::shared_ptr> + proactive_resources_; + absl::node_hash_map actions_; Event::ScaledTimerTypeMapConstSharedPtr timer_minimums_; diff --git a/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc b/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc index 24b451ee6fe72..00d8e42665f64 100644 --- a/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc +++ b/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc @@ -20,7 +20,7 @@ class MockMemoryStatsReader : public MemoryStatsReader { MOCK_METHOD(uint64_t, unmappedHeapBytes, ()); }; -class ResourcePressure : public Server::ResourceMonitor::Callbacks { +class ResourcePressure : public Server::ResourceUpdateCallbacks { public: void onSuccess(const Server::ResourceUsage& usage) override { pressure_ = usage.resource_pressure_; diff --git a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_test.cc b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_test.cc index 5d05210ad4f11..7010e02241b5c 100644 --- a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_test.cc +++ b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_test.cc @@ -38,7 +38,7 @@ class TestableInjectedResourceMonitor : public InjectedResourceMonitor { Event::Dispatcher& dispatcher_; }; -class MockedCallbacks : public Server::ResourceMonitor::Callbacks { +class MockedCallbacks : public Server::ResourceUpdateCallbacks { public: MOCK_METHOD(void, onSuccess, (const Server::ResourceUsage&)); MOCK_METHOD(void, onFailure, (const EnvoyException&)); diff --git a/test/integration/fake_resource_monitor.cc b/test/integration/fake_resource_monitor.cc index 85a11084f6e14..eb57a5ba66699 100644 --- a/test/integration/fake_resource_monitor.cc +++ b/test/integration/fake_resource_monitor.cc @@ -4,7 +4,7 @@ namespace Envoy { FakeResourceMonitor::~FakeResourceMonitor() { factory_.onMonitorDestroyed(this); } -void FakeResourceMonitor::updateResourceUsage(Callbacks& callbacks) { +void FakeResourceMonitor::updateResourceUsage(Server::ResourceUpdateCallbacks& callbacks) { Server::ResourceUsage usage; usage.resource_pressure_ = pressure_; callbacks.onSuccess(usage); diff --git a/test/integration/fake_resource_monitor.h b/test/integration/fake_resource_monitor.h index 84d40eb70dcb9..b32b581912d1b 100644 --- a/test/integration/fake_resource_monitor.h +++ b/test/integration/fake_resource_monitor.h @@ -15,7 +15,7 @@ class FakeResourceMonitor : public Server::ResourceMonitor { : dispatcher_(dispatcher), factory_(factory), pressure_(0.0) {} // Server::ResourceMonitor ~FakeResourceMonitor() override; - void updateResourceUsage(Callbacks& callbacks) override; + void updateResourceUsage(Server::ResourceUpdateCallbacks& callbacks) override; void setResourcePressure(double pressure) { dispatcher_.post([this, pressure] { pressure_ = pressure; }); diff --git a/test/mocks/server/overload_manager.cc b/test/mocks/server/overload_manager.cc index 4ebf19f032b88..9ca2c36a87557 100644 --- a/test/mocks/server/overload_manager.cc +++ b/test/mocks/server/overload_manager.cc @@ -10,11 +10,15 @@ namespace Envoy { namespace Server { +using ::testing::Return; using ::testing::ReturnRef; MockThreadLocalOverloadState::MockThreadLocalOverloadState() : disabled_state_(OverloadActionState::inactive()) { ON_CALL(*this, getState).WillByDefault(ReturnRef(disabled_state_)); + ON_CALL(*this, tryAllocateResource).WillByDefault(Return(true)); + ON_CALL(*this, tryDeallocateResource).WillByDefault(Return(true)); + ON_CALL(*this, isResourceMonitorEnabled).WillByDefault(Return(false)); } MockOverloadManager::MockOverloadManager() { diff --git a/test/mocks/server/overload_manager.h b/test/mocks/server/overload_manager.h index e5ab03992836f..c3c08c8895ebd 100644 --- a/test/mocks/server/overload_manager.h +++ b/test/mocks/server/overload_manager.h @@ -14,6 +14,9 @@ class MockThreadLocalOverloadState : public ThreadLocalOverloadState { public: MockThreadLocalOverloadState(); MOCK_METHOD(const OverloadActionState&, getState, (const std::string&), (override)); + MOCK_METHOD(bool, tryAllocateResource, (OverloadProactiveResourceName, int64_t)); + MOCK_METHOD(bool, tryDeallocateResource, (OverloadProactiveResourceName, int64_t)); + MOCK_METHOD(bool, isResourceMonitorEnabled, (OverloadProactiveResourceName)); private: const OverloadActionState disabled_state_; diff --git a/test/server/overload_manager_impl_test.cc b/test/server/overload_manager_impl_test.cc index 0886e0181d5ff..6fa231cfeb715 100644 --- a/test/server/overload_manager_impl_test.cc +++ b/test/server/overload_manager_impl_test.cc @@ -57,7 +57,7 @@ class FakeResourceMonitor : public ResourceMonitor { update_async_ = new_update_async; } - void updateResourceUsage(ResourceMonitor::Callbacks& callbacks) override { + void updateResourceUsage(ResourceUpdateCallbacks& callbacks) override { if (update_async_) { callbacks_.emplace(callbacks); } else { @@ -74,7 +74,7 @@ class FakeResourceMonitor : public ResourceMonitor { } private: - void publishUpdate(ResourceMonitor::Callbacks& callbacks) { + void publishUpdate(ResourceUpdateCallbacks& callbacks) { if (absl::holds_alternative(response_)) { Server::ResourceUsage usage; usage.resource_pressure_ = absl::get(response_); @@ -88,7 +88,39 @@ class FakeResourceMonitor : public ResourceMonitor { Event::Dispatcher& dispatcher_; absl::variant response_; bool update_async_ = false; - absl::optional> callbacks_; + absl::optional> callbacks_; +}; + +class FakeProactiveResourceMonitor : public ProactiveResourceMonitor { +public: + FakeProactiveResourceMonitor(uint64_t max) : max_(max), current_(0){}; + + bool tryAllocateResource(int64_t increment) override { + int64_t new_val = (current_ += increment); + if (new_val > static_cast(max_) || new_val < 0) { + current_ -= increment; + return false; + } + return true; + } + + bool tryDeallocateResource(int64_t decrement) override { + RELEASE_ASSERT(decrement <= current_, + "Cannot deallocate resource, current resource usage is lower than decrement"); + int64_t new_val = (current_ -= decrement); + if (new_val < 0) { + current_ += decrement; + return false; + } + return true; + } + + int64_t currentResourceUsage() const override { return current_.load(); } + int64_t maxResourceUsage() const override { return max_; } + +private: + int64_t max_; + std::atomic current_; }; template @@ -114,6 +146,30 @@ class FakeResourceMonitorFactory : public Server::Configuration::ResourceMonitor const std::string name_; }; +template +class FakeProactiveResourceMonitorFactory + : public Server::Configuration::ProactiveResourceMonitorFactory { +public: + FakeProactiveResourceMonitorFactory(const std::string& name) : monitor_(nullptr), name_(name) {} + + Server::ProactiveResourceMonitorPtr + createProactiveResourceMonitor(const Protobuf::Message&, + Server::Configuration::ResourceMonitorFactoryContext&) override { + auto monitor = std::make_unique(3); + monitor_ = monitor.get(); + return monitor; + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return ProtobufTypes::MessagePtr{new ConfigType()}; + } + + std::string name() const override { return name_; } + + FakeProactiveResourceMonitor* monitor_; // not owned + const std::string name_; +}; + class TestOverloadManager : public OverloadManagerImpl { public: TestOverloadManager(Event::Dispatcher& dispatcher, Stats::Scope& stats_scope, @@ -146,8 +202,10 @@ class OverloadManagerImplTest : public testing::Test { : factory1_("envoy.resource_monitors.fake_resource1"), factory2_("envoy.resource_monitors.fake_resource2"), factory3_("envoy.resource_monitors.fake_resource3"), - factory4_("envoy.resource_monitors.fake_resource4"), register_factory1_(factory1_), - register_factory2_(factory2_), register_factory3_(factory3_), register_factory4_(factory4_), + factory4_("envoy.resource_monitors.fake_resource4"), + factory5_("envoy.resource_monitors.global_downstream_max_connections"), + register_factory1_(factory1_), register_factory2_(factory2_), register_factory3_(factory3_), + register_factory4_(factory4_), register_factory5_(factory5_), api_(Api::createApiForTest(stats_)) {} void setDispatcherExpectation() { @@ -174,10 +232,12 @@ class OverloadManagerImplTest : public testing::Test { FakeResourceMonitorFactory factory2_; FakeResourceMonitorFactory factory3_; FakeResourceMonitorFactory factory4_; + FakeProactiveResourceMonitorFactory factory5_; Registry::InjectFactory register_factory1_; Registry::InjectFactory register_factory2_; Registry::InjectFactory register_factory3_; Registry::InjectFactory register_factory4_; + Registry::InjectFactory register_factory5_; NiceMock dispatcher_; NiceMock* timer_; // not owned Stats::TestUtil::TestStore stats_; @@ -215,6 +275,20 @@ constexpr char kRegularStateConfig[] = R"YAML( saturation_threshold: 0.8 )YAML"; +constexpr char proactiveResourceConfig[] = R"YAML( + refresh_interval: + seconds: 1 + resource_monitors: + - name: envoy.resource_monitors.fake_resource1 + - name: envoy.resource_monitors.global_downstream_max_connections + actions: + - name: envoy.overload_actions.dummy_action + triggers: + - name: envoy.resource_monitors.fake_resource1 + threshold: + value: 0.9 +)YAML"; + TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { setDispatcherExpectation(); @@ -230,6 +304,9 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { [&](OverloadActionState) { EXPECT_TRUE(false); }); manager->start(); + EXPECT_FALSE(manager->getThreadLocalOverloadState().isResourceMonitorEnabled( + OverloadProactiveResourceName::GlobalDownstreamMaxConnections)); + Stats::Gauge& active_gauge = stats_.gauge("overload.envoy.overload_actions.dummy_action.active", Stats::Gauge::ImportMode::Accumulate); Stats::Gauge& scale_percent_gauge = @@ -566,6 +643,17 @@ TEST_F(OverloadManagerImplTest, DuplicateResourceMonitor) { "Duplicate resource monitor .*"); } +TEST_F(OverloadManagerImplTest, DuplicateProactiveResourceMonitor) { + const std::string config = R"EOF( + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + - name: "envoy.resource_monitors.global_downstream_max_connections" + )EOF"; + + EXPECT_THROW_WITH_REGEX(createOverloadManager(config), EnvoyException, + "Duplicate resource monitor .*"); +} + TEST_F(OverloadManagerImplTest, DuplicateOverloadAction) { const std::string config = R"EOF( actions: @@ -711,6 +799,32 @@ TEST_F(OverloadManagerImplTest, Shutdown) { manager->stop(); } +TEST_F(OverloadManagerImplTest, ProactiveResourceAllocateAndDeallocateResourceTest) { + setDispatcherExpectation(); + auto manager(createOverloadManager(proactiveResourceConfig)); + Stats::Counter& failed_updates = + stats_.counter("overload.envoy.resource_monitors.global_downstream_max_connections." + "failed_updates"); + manager->start(); + EXPECT_TRUE(manager->getThreadLocalOverloadState().isResourceMonitorEnabled( + OverloadProactiveResourceName::GlobalDownstreamMaxConnections)); + bool resource_allocated = manager->getThreadLocalOverloadState().tryAllocateResource( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections, 1); + EXPECT_TRUE(resource_allocated); + resource_allocated = manager->getThreadLocalOverloadState().tryAllocateResource( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections, 3); + EXPECT_FALSE(resource_allocated); + EXPECT_EQ(1, failed_updates.value()); + + bool resource_deallocated = manager->getThreadLocalOverloadState().tryDeallocateResource( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections, 1); + EXPECT_TRUE(resource_deallocated); + EXPECT_DEATH(manager->getThreadLocalOverloadState().tryDeallocateResource( + Server::OverloadProactiveResourceName::GlobalDownstreamMaxConnections, 1), + ".*Cannot deallocate resource, current resource usage is lower than decrement.*"); + manager->stop(); +} + } // namespace } // namespace Server } // namespace Envoy From 94d0fdde196067b2108231bedad4da035b8f61c3 Mon Sep 17 00:00:00 2001 From: Saidi Tang Date: Thu, 2 Dec 2021 10:30:29 -0500 Subject: [PATCH 45/51] test: fix headerMapEqualIgnoreOrder utility missing a check (#19158) * Added missing check outside lambda Signed-off-by: tangsaidi --- .../simple_http_cache_test.cc | 35 ++++++++++++++----- .../ext_proc/ext_proc_integration_test.cc | 8 ++--- .../extensions/filters/http/ext_proc/utils.cc | 9 ++++- test/extensions/filters/http/ext_proc/utils.h | 5 +++ test/test_common/utility.cc | 7 ++-- 5 files changed, 48 insertions(+), 16 deletions(-) diff --git a/test/extensions/filters/http/cache/simple_http_cache/simple_http_cache_test.cc b/test/extensions/filters/http/cache/simple_http_cache/simple_http_cache_test.cc index f2eb5ac181af0..31b981ec50c3b 100644 --- a/test/extensions/filters/http/cache/simple_http_cache/simple_http_cache_test.cc +++ b/test/extensions/filters/http/cache/simple_http_cache/simple_http_cache_test.cc @@ -343,17 +343,25 @@ TEST_F(SimpleHttpCacheTest, UpdateHeadersAndMetadata) { const std::string time_value_1 = formatter_.fromTime(time_source_.systemTime()); Http::TestResponseHeaderMapImpl response_headers{{"date", time_value_1}, {"cache-control", "public,max-age=3600"}}; + Http::TestResponseHeaderMapImpl response_headers_with_age(response_headers); + response_headers_with_age.setReferenceKey(Http::LowerCaseString("age"), "0"); + insert(request_path_1, response_headers, "body"); - EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers)); + EXPECT_TRUE( + expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_with_age)); // Update the date field in the headers time_source_.advanceTimeWait(Seconds(3601)); const SystemTime time_2 = time_source_.systemTime(); const std::string time_value_2 = formatter_.fromTime(time_2); - response_headers = Http::TestResponseHeaderMapImpl{{"date", time_value_2}, - {"cache-control", "public,max-age=3600"}}; - updateHeaders(request_path_1, response_headers, {time_2}); - EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers)); + Http::TestResponseHeaderMapImpl response_headers_2 = Http::TestResponseHeaderMapImpl{ + {"date", time_value_2}, {"cache-control", "public,max-age=3600"}}; + Http::TestResponseHeaderMapImpl response_headers_with_age_2(response_headers_2); + response_headers_with_age_2.setReferenceKey(Http::LowerCaseString("age"), "0"); + + updateHeaders(request_path_1, response_headers_2, {time_2}); + EXPECT_TRUE( + expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_with_age_2)); } TEST_F(SimpleHttpCacheTest, UpdateHeadersForMissingKey) { @@ -373,6 +381,8 @@ TEST_F(SimpleHttpCacheTest, UpdateHeadersDisabledForVaryHeaders) { {"accept", "image/*"}, {"vary", "accept"}}; insert(request_path_1, response_headers_1, "body"); + // An age header is inserted by `makeLookUpResult` + response_headers_1.setReferenceKey(Http::LowerCaseString("age"), "0"); EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_1)); // Update the date field in the headers @@ -384,7 +394,8 @@ TEST_F(SimpleHttpCacheTest, UpdateHeadersDisabledForVaryHeaders) { {"accept", "image/*"}, {"vary", "accept"}}; updateHeaders(request_path_1, response_headers_2, {time_2}); - + response_headers_1.setReferenceKey(Http::LowerCaseString("age"), "3600"); + // the age is still 0 because an entry is considered fresh after validation EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_1)); } @@ -394,6 +405,8 @@ TEST_F(SimpleHttpCacheTest, UpdateHeadersSkipEtagHeader) { Http::TestResponseHeaderMapImpl response_headers_1{ {"date", time_value_1}, {"cache-control", "public,max-age=3600"}, {"etag", "0000-0000"}}; insert(request_path_1, response_headers_1, "body"); + // An age header is inserted by `makeLookUpResult` + response_headers_1.setReferenceKey(Http::LowerCaseString("age"), "0"); EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_1)); // Update the date field in the headers @@ -407,7 +420,7 @@ TEST_F(SimpleHttpCacheTest, UpdateHeadersSkipEtagHeader) { {"date", time_value_2}, {"cache-control", "public,max-age=3600"}, {"etag", "0000-0000"}}; updateHeaders(request_path_1, response_headers_2, {time_2}); - + response_headers_3.setReferenceKey(Http::LowerCaseString("age"), "0"); EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_3)); } @@ -425,6 +438,9 @@ TEST_F(SimpleHttpCacheTest, UpdateHeadersSkipSpecificHeaders) { {"etag", "1111-1111"}, {"link", "; rel=\"preconnect\""}}; insert(request_path_1, origin_response_headers, "body"); + + // An age header is inserted by `makeLookUpResult` + origin_response_headers.setReferenceKey(Http::LowerCaseString("age"), "0"); EXPECT_TRUE( expectLookupSuccessWithHeaders(lookup(request_path_1).get(), origin_response_headers)); time_source_.advanceTimeWait(Seconds(100)); @@ -454,7 +470,6 @@ TEST_F(SimpleHttpCacheTest, UpdateHeadersSkipSpecificHeaders) { {"link", "; rel=\"preconnect\""}}; updateHeaders(request_path_1, incoming_response_headers, {time_2}); - EXPECT_TRUE( expectLookupSuccessWithHeaders(lookup(request_path_1).get(), expected_response_headers)); } @@ -471,6 +486,9 @@ TEST_F(SimpleHttpCacheTest, UpdateHeadersWithMultivalue) { {"link", "; rel=\"preconnect\""}, {"link", "; rel=\"preconnect\""}}; insert(request_path_1, response_headers_1, "body"); + + // An age header is inserted by `makeLookUpResult` + response_headers_1.setReferenceKey(Http::LowerCaseString("age"), "0"); EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_1)); Http::TestResponseHeaderMapImpl response_headers_2{ @@ -481,6 +499,7 @@ TEST_F(SimpleHttpCacheTest, UpdateHeadersWithMultivalue) { updateHeaders(request_path_1, response_headers_2, {time_1}); + response_headers_2.setReferenceKey(Http::LowerCaseString("age"), "0"); EXPECT_TRUE(expectLookupSuccessWithHeaders(lookup(request_path_1).get(), response_headers_2)); } diff --git a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc index aeffd710edc20..661ebdc6d72b3 100644 --- a/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc +++ b/test/extensions/filters/http/ext_proc/ext_proc_integration_test.cc @@ -447,11 +447,9 @@ TEST_P(ExtProcIntegrationTest, GetAndSetHeaders) { [](Http::HeaderMap& headers) { headers.addCopy(LowerCaseString("x-remove-this"), "yes"); }); processRequestHeadersMessage(true, [](const HttpHeaders& headers, HeadersResponse& headers_resp) { - Http::TestRequestHeaderMapImpl expected_request_headers{{":scheme", "http"}, - {":method", "GET"}, - {"host", "host"}, - {":path", "/"}, - {"x-remove-this", "yes"}}; + Http::TestRequestHeaderMapImpl expected_request_headers{ + {":scheme", "http"}, {":method", "GET"}, {"host", "host"}, + {":path", "/"}, {"x-remove-this", "yes"}, {"x-forwarded-proto", "http"}}; EXPECT_THAT(headers.headers(), HeaderProtosEqual(expected_request_headers)); auto response_header_mutation = headers_resp.mutable_response()->mutable_header_mutation(); diff --git a/test/extensions/filters/http/ext_proc/utils.cc b/test/extensions/filters/http/ext_proc/utils.cc index c9bb93b4738d5..4bc448909b184 100644 --- a/test/extensions/filters/http/ext_proc/utils.cc +++ b/test/extensions/filters/http/ext_proc/utils.cc @@ -7,13 +7,20 @@ namespace Extensions { namespace HttpFilters { namespace ExternalProcessing { +const absl::flat_hash_set ExtProcTestUtility::ignoredHeaders() { + CONSTRUCT_ON_FIRST_USE(absl::flat_hash_set, "x-request-id", + "x-envoy-upstream-service-time"); +} + bool ExtProcTestUtility::headerProtosEqualIgnoreOrder( const Http::HeaderMap& expected, const envoy::config::core::v3::HeaderMap& actual) { // Comparing header maps is hard because they have duplicates in them. // So we're going to turn them into a HeaderMap and let Envoy do the work. Http::TestRequestHeaderMapImpl actual_headers; for (const auto& header : actual.headers()) { - actual_headers.addCopy(header.key(), header.value()); + if (!ignoredHeaders().contains(header.key())) { + actual_headers.addCopy(header.key(), header.value()); + } } return TestUtility::headerMapEqualIgnoreOrder(expected, actual_headers); } diff --git a/test/extensions/filters/http/ext_proc/utils.h b/test/extensions/filters/http/ext_proc/utils.h index 98274370d9451..ad6b0f4af1044 100644 --- a/test/extensions/filters/http/ext_proc/utils.h +++ b/test/extensions/filters/http/ext_proc/utils.h @@ -16,6 +16,11 @@ class ExtProcTestUtility { // Compare a reference header map to a proto static bool headerProtosEqualIgnoreOrder(const Http::HeaderMap& expected, const envoy::config::core::v3::HeaderMap& actual); + +private: + // These headers are present in the actual, but cannot be specified in the expected + // ignoredHeaders should not be used for equal comparison + static const absl::flat_hash_set ignoredHeaders(); }; MATCHER_P(HeaderProtosEqual, expected, "HTTP header protos match") { diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 865d84f3aa05c..0fa076fe35d0c 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -74,7 +74,9 @@ bool TestUtility::headerMapEqualIgnoreOrder(const Http::HeaderMap& lhs, lhs_keys.insert(key); return Http::HeaderMap::Iterate::Continue; }); - rhs.iterate([&lhs, &rhs, &rhs_keys](const Http::HeaderEntry& header) -> Http::HeaderMap::Iterate { + bool values_match = true; + rhs.iterate([&values_match, &lhs, &rhs, + &rhs_keys](const Http::HeaderEntry& header) -> Http::HeaderMap::Iterate { const std::string key{header.key().getStringView()}; // Compare with canonicalized multi-value headers. This ensures we respect order within // a header. @@ -84,12 +86,13 @@ bool TestUtility::headerMapEqualIgnoreOrder(const Http::HeaderMap& lhs, Http::HeaderUtility::getAllOfHeaderAsString(rhs, Http::LowerCaseString(key)); ASSERT(rhs_entry.result()); if (lhs_entry.result() != rhs_entry.result()) { + values_match = false; return Http::HeaderMap::Iterate::Break; } rhs_keys.insert(key); return Http::HeaderMap::Iterate::Continue; }); - return lhs_keys.size() == rhs_keys.size(); + return values_match && lhs_keys.size() == rhs_keys.size(); } bool TestUtility::buffersEqual(const Buffer::Instance& lhs, const Buffer::Instance& rhs) { From 2e0ad0beb9625ccedaec8f3093aa5e6e1dc484f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Thu, 2 Dec 2021 15:25:22 -0500 Subject: [PATCH 46/51] thrift: don't close the downstream on an upstream overflow (#19133) When we fail to get an upstream connection (e.g.: PoolFailureReason::Overflow) there's no need to close the downstream connection, since the request never made it through. So we keep it open and avoid an issue that happens when closing remote connections after a local response - see below. Risk Level: low Testing: updated unit tests Docs Changes: n/a Release Notes: added Signed-off-by: Raul Gutierrez Segales --- docs/root/version_history/current.rst | 1 + .../filters/network/thrift_proxy/router/upstream_request.cc | 2 +- test/extensions/filters/network/thrift_proxy/router_test.cc | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 8a6bd863c3ef8..1baf70bdf6956 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -28,6 +28,7 @@ Bug Fixes * ext_authz: fix the ext_authz http filter to correctly set response flags to ``UAEX`` when a connection is denied. * ext_authz: fix the ext_authz network filter to correctly set response flag and code details to ``UAEX`` when a connection is denied. * listener: fixed issue where more than one listener could listen on the same port if using reuse port, thus randomly accepting connections on different listeners. This configuration is now rejected. +* thrift_proxy: do not close downstream connections when an upstream connection overflow happens. * thrift_proxy: fix the thrift_proxy connection manager to correctly report success/error response metrics when performing :ref:`payload passthrough `. Removed Config or Runtime diff --git a/source/extensions/filters/network/thrift_proxy/router/upstream_request.cc b/source/extensions/filters/network/thrift_proxy/router/upstream_request.cc index b1cededafe95a..6947a9856d0e8 100644 --- a/source/extensions/filters/network/thrift_proxy/router/upstream_request.cc +++ b/source/extensions/filters/network/thrift_proxy/router/upstream_request.cc @@ -270,7 +270,7 @@ void UpstreamRequest::onResetStream(ConnectionPool::PoolFailureReason reason) { stats_.incResponseLocalException(parent_.cluster()); parent_.sendLocalReply(AppException(AppExceptionType::InternalError, "thrift upstream request: too many connections"), - true); + false /* Don't close the downstream connection. */); break; case ConnectionPool::PoolFailureReason::LocalConnectionFailure: upstream_host_->outlierDetector().putResult( diff --git a/test/extensions/filters/network/thrift_proxy/router_test.cc b/test/extensions/filters/network/thrift_proxy/router_test.cc index c001013f01886..199a261210c7d 100644 --- a/test/extensions/filters/network/thrift_proxy/router_test.cc +++ b/test/extensions/filters/network/thrift_proxy/router_test.cc @@ -759,7 +759,7 @@ TEST_F(ThriftRouterTest, PoolOverflowFailure) { auto& app_ex = dynamic_cast(response); EXPECT_EQ(AppExceptionType::InternalError, app_ex.type_); EXPECT_THAT(app_ex.what(), ContainsRegex(".*too many connections.*")); - EXPECT_TRUE(end_stream); + EXPECT_FALSE(end_stream); })); context_.cluster_manager_.thread_local_cluster_.tcp_conn_pool_.poolFailure( ConnectionPool::PoolFailureReason::Overflow, true); From 904f88a4d3461b3f0306594a169aaf953d1f290e Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 2 Dec 2021 15:37:12 -0500 Subject: [PATCH 47/51] test: adding HTTP/3 fail socket write test (#19171) Signed-off-by: Alyssa Wilk --- test/integration/BUILD | 1 + .../buffer_accounting_integration_test.cc | 30 ++++++++--------- .../filters/test_socket_interface.cc | 26 +++++++++++---- .../filters/test_socket_interface.h | 32 +++++++++++-------- .../http2_flood_integration_test.cc | 32 +++++++++---------- test/integration/protocol_integration_test.cc | 32 +++++++++++++++---- test/integration/socket_interface_swap.cc | 13 ++++---- test/integration/socket_interface_swap.h | 12 +++---- 8 files changed, 107 insertions(+), 71 deletions(-) diff --git a/test/integration/BUILD b/test/integration/BUILD index a1348059a050b..cc888242847f2 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -515,6 +515,7 @@ envoy_cc_test_library( "//test/integration/filters:random_pause_filter_lib", "//test/integration/filters:remove_response_headers_lib", "//test/test_common:logging_lib", + "//test/test_common:threadsafe_singleton_injector_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", diff --git a/test/integration/buffer_accounting_integration_test.cc b/test/integration/buffer_accounting_integration_test.cc index 13f68cc08a40f..d996dbdf601f4 100644 --- a/test/integration/buffer_accounting_integration_test.cc +++ b/test/integration/buffer_accounting_integration_test.cc @@ -213,8 +213,8 @@ TEST_P(Http2BufferWatermarksTest, ShouldTrackAllocatedBytesToUpstream) { buffer_factory_->setExpectedAccountBalance(request_body_size, num_requests); // Makes us have Envoy's writes to upstream return EAGAIN - writev_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); + write_matcher_->setWriteReturnsEgain(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -228,7 +228,7 @@ TEST_P(Http2BufferWatermarksTest, ShouldTrackAllocatedBytesToUpstream) { << " buffer max: " << buffer_factory_->maxBufferSize() << printAccounts(); } - writev_matcher_->setResumeWrites(); + write_matcher_->setResumeWrites(); for (auto& response : responses) { ASSERT_TRUE(response->waitForEndStream()); @@ -246,12 +246,12 @@ TEST_P(Http2BufferWatermarksTest, ShouldTrackAllocatedBytesToDownstream) { initialize(); buffer_factory_->setExpectedAccountBalance(response_body_size, num_requests); - writev_matcher_->setSourcePort(lookupPort("http")); + write_matcher_->setSourcePort(lookupPort("http")); codec_client_ = makeHttpConnection(lookupPort("http")); // Simulate TCP push back on the Envoy's downstream network socket, so that outbound frames // start to accumulate in the transport socket buffer. - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); auto responses = sendRequests(num_requests, request_body_size, response_body_size); @@ -263,7 +263,7 @@ TEST_P(Http2BufferWatermarksTest, ShouldTrackAllocatedBytesToDownstream) { << " buffer max: " << buffer_factory_->maxBufferSize() << printAccounts(); } - writev_matcher_->setResumeWrites(); + write_matcher_->setResumeWrites(); // Wait for streams to terminate. for (auto& response : responses) { @@ -454,8 +454,8 @@ TEST_P(Http2OverloadManagerIntegrationTest, initialize(); // Makes us have Envoy's writes to upstream return EAGAIN - writev_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); + write_matcher_->setWriteReturnsEgain(); codec_client_ = makeHttpConnection(lookupPort("http")); auto smallest_request_response = std::move(sendRequests(1, 4096, 4096)[0]); @@ -500,7 +500,7 @@ TEST_P(Http2OverloadManagerIntegrationTest, "overload.envoy.overload_actions.reset_high_memory_stream.scale_percent", 0); // Resume writes to upstream, any request streams that survive can go through. - writev_matcher_->setResumeWrites(); + write_matcher_->setResumeWrites(); if (!streamBufferAccounting()) { // If we're not doing the accounting, we didn't end up resetting these @@ -533,9 +533,9 @@ TEST_P(Http2OverloadManagerIntegrationTest, initialize(); // Makes us have Envoy's writes to downstream return EAGAIN - writev_matcher_->setSourcePort(lookupPort("http")); + write_matcher_->setSourcePort(lookupPort("http")); codec_client_ = makeHttpConnection(lookupPort("http")); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); auto smallest_response = std::move(sendRequests(1, 10, 4096)[0]); waitForNextUpstreamRequest(); @@ -589,7 +589,7 @@ TEST_P(Http2OverloadManagerIntegrationTest, "overload.envoy.overload_actions.reset_high_memory_stream.scale_percent", 0); // Resume writes to downstream, any responses that survive can go through. - writev_matcher_->setResumeWrites(); + write_matcher_->setResumeWrites(); if (streamBufferAccounting()) { EXPECT_TRUE(largest_response->waitForReset()); @@ -642,8 +642,8 @@ TEST_P(Http2OverloadManagerIntegrationTest, CanResetStreamIfEnvoyLevelStreamEnde codec_client_ = makeRawHttpConnection(makeClientConnection(lookupPort("http")), http2_options); // Makes us have Envoy's writes to downstream return EAGAIN - writev_matcher_->setSourcePort(lookupPort("http")); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setSourcePort(lookupPort("http")); + write_matcher_->setWriteReturnsEgain(); // Send a request auto encoder_decoder = codec_client_->startRequest(Http::TestRequestHeaderMapImpl{ @@ -690,7 +690,7 @@ TEST_P(Http2OverloadManagerIntegrationTest, CanResetStreamIfEnvoyLevelStreamEnde "overload.envoy.overload_actions.reset_high_memory_stream.scale_percent", 0); // Resume writes to downstream. - writev_matcher_->setResumeWrites(); + write_matcher_->setResumeWrites(); if (streamBufferAccounting()) { EXPECT_TRUE(response->waitForReset()); diff --git a/test/integration/filters/test_socket_interface.cc b/test/integration/filters/test_socket_interface.cc index 5991e7a948708..3356cecc87612 100644 --- a/test/integration/filters/test_socket_interface.cc +++ b/test/integration/filters/test_socket_interface.cc @@ -13,10 +13,22 @@ namespace Envoy { namespace Network { +Api::IoCallUint64Result TestIoSocketHandle::sendmsg(const Buffer::RawSlice* slices, + uint64_t num_slice, int flags, + const Address::Ip* self_ip, + const Address::Instance& peer_address) { + if (write_override_) { + auto result = write_override_(this, slices, num_slice); + if (result.has_value()) { + return std::move(result).value(); + } + } + return Test::IoSocketHandlePlatformImpl::sendmsg(slices, num_slice, flags, self_ip, peer_address); +} Api::IoCallUint64Result TestIoSocketHandle::writev(const Buffer::RawSlice* slices, uint64_t num_slice) { - if (writev_override_) { - auto result = writev_override_(this, slices, num_slice); + if (write_override_) { + auto result = write_override_(this, slices, num_slice); if (result.has_value()) { return std::move(result).value(); } @@ -30,8 +42,8 @@ IoHandlePtr TestIoSocketHandle::accept(struct sockaddr* addr, socklen_t* addrlen return nullptr; } - return std::make_unique(writev_override_, result.return_value_, - socket_v6only_, domain_); + return std::make_unique(write_override_, result.return_value_, socket_v6only_, + domain_); } IoHandlePtr TestIoSocketHandle::duplicate() { @@ -40,13 +52,13 @@ IoHandlePtr TestIoSocketHandle::duplicate() { throw EnvoyException(fmt::format("duplicate failed for '{}': ({}) {}", fd_, result.errno_, errorDetails(result.errno_))); } - return std::make_unique(writev_override_, result.return_value_, - socket_v6only_, domain_); + return std::make_unique(write_override_, result.return_value_, socket_v6only_, + domain_); } IoHandlePtr TestSocketInterface::makeSocket(int socket_fd, bool socket_v6only, absl::optional domain) const { - return std::make_unique(writev_override_proc_, socket_fd, socket_v6only, + return std::make_unique(write_override_proc_, socket_fd, socket_v6only, domain); } diff --git a/test/integration/filters/test_socket_interface.h b/test/integration/filters/test_socket_interface.h index 61da3e86fa105..abb48f77da9eb 100644 --- a/test/integration/filters/test_socket_interface.h +++ b/test/integration/filters/test_socket_interface.h @@ -21,15 +21,15 @@ namespace Network { class TestIoSocketHandle : public Test::IoSocketHandlePlatformImpl { public: - using WritevOverrideType = absl::optional(TestIoSocketHandle* io_handle, - const Buffer::RawSlice* slices, - uint64_t num_slice); - using WritevOverrideProc = std::function; + using WriteOverrideType = absl::optional(TestIoSocketHandle* io_handle, + const Buffer::RawSlice* slices, + uint64_t num_slice); + using WriteOverrideProc = std::function; - TestIoSocketHandle(WritevOverrideProc writev_override_proc, os_fd_t fd = INVALID_SOCKET, + TestIoSocketHandle(WriteOverrideProc write_override_proc, os_fd_t fd = INVALID_SOCKET, bool socket_v6only = false, absl::optional domain = absl::nullopt) : Test::IoSocketHandlePlatformImpl(fd, socket_v6only, domain), - writev_override_(writev_override_proc) {} + write_override_(write_override_proc) {} void initializeFileEvent(Event::Dispatcher& dispatcher, Event::FileReadyCb cb, Event::FileTriggerType trigger, uint32_t events) override { @@ -50,9 +50,13 @@ class TestIoSocketHandle : public Test::IoSocketHandlePlatformImpl { private: IoHandlePtr accept(struct sockaddr* addr, socklen_t* addrlen) override; Api::IoCallUint64Result writev(const Buffer::RawSlice* slices, uint64_t num_slice) override; + Api::IoCallUint64Result sendmsg(const Buffer::RawSlice* slices, uint64_t num_slice, int flags, + const Address::Ip* self_ip, + const Address::Instance& peer_address) override; + IoHandlePtr duplicate() override; - const WritevOverrideProc writev_override_; + const WriteOverrideProc write_override_; absl::Mutex mutex_; Event::Dispatcher* dispatcher_ ABSL_GUARDED_BY(mutex_) = nullptr; }; @@ -68,22 +72,22 @@ class TestIoSocketHandle : public Test::IoSocketHandlePlatformImpl { class TestSocketInterface : public SocketInterfaceImpl { public: /** - * Override the behavior of the IoSocketHandleImpl::writev() method. - * The supplied callback is invoked with the arguments of the writev method and the index + * Override the behavior of the IoSocketHandleImpl::writev() and + * IoSocketHandleImpl::sendmsg() methods. + * The supplied callback is invoked with the slices arguments of the write method and the index * of the accepted socket. * Returning absl::nullopt from the callback continues normal execution of the - * IoSocketHandleImpl::writev() method. Returning a Api::IoCallUint64Result from callback skips - * the IoSocketHandleImpl::writev() with the returned result value. + * write methods. Returning a Api::IoCallUint64Result from callback skips + * the write methods with the returned result value. */ - TestSocketInterface(TestIoSocketHandle::WritevOverrideProc writev) - : writev_override_proc_(writev) {} + TestSocketInterface(TestIoSocketHandle::WriteOverrideProc write) : write_override_proc_(write) {} private: // SocketInterfaceImpl IoHandlePtr makeSocket(int socket_fd, bool socket_v6only, absl::optional domain) const override; - const TestIoSocketHandle::WritevOverrideProc writev_override_proc_; + const TestIoSocketHandle::WriteOverrideProc write_override_proc_; }; } // namespace Network diff --git a/test/integration/http2_flood_integration_test.cc b/test/integration/http2_flood_integration_test.cc index 848ad5b9964f4..9950e49d58cf7 100644 --- a/test/integration/http2_flood_integration_test.cc +++ b/test/integration/http2_flood_integration_test.cc @@ -119,7 +119,7 @@ void Http2FloodMitigationTest::beginSession() { options->emplace_back(std::make_shared( envoy::config::core::v3::SocketOption::STATE_PREBIND, ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_RCVBUF), 1024)); - writev_matcher_->setSourcePort(lookupPort("http")); + write_matcher_->setSourcePort(lookupPort("http")); tcp_client_ = makeTcpConnection(lookupPort("http"), options); startHttp2Session(); } @@ -156,11 +156,11 @@ void Http2FloodMitigationTest::floodClient(const Http2Frame& frame, uint32_t num waitForNextUpstreamRequest(); // Make Envoy's writes into the upstream connection to return EAGAIN - writev_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); + write_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); auto buf = serializeFrames(frame, num_frames); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); auto* upstream = fake_upstreams_.front().get(); ASSERT_TRUE(upstream->rawWriteConnection(0, std::string(buf.begin(), buf.end()))); @@ -184,7 +184,7 @@ void Http2FloodMitigationTest::floodServer(absl::string_view host, absl::string_ auto frame = readFrame(); EXPECT_EQ(Http2Frame::Type::Headers, frame.type()); EXPECT_EQ(expected_http_status, frame.responseStatus()); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); for (uint32_t frame = 0; frame < num_frames; ++frame) { request = Http2Frame::makeRequest(Http2Frame::makeClientStreamId(++request_idx), host, path); sendFrame(request); @@ -210,7 +210,7 @@ void Http2FloodMitigationTest::prefillOutboundDownstreamQueue(uint32_t data_fram // such the next response triggers flood protection. // Simulate TCP push back on the Envoy's downstream network socket, so that outbound frames // start to accumulate in the transport socket buffer. - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); const auto request = Http2Frame::makeRequest( Http2Frame::makeClientStreamId(0), "host", "/test/long/url", @@ -258,11 +258,11 @@ Http2FloodMitigationTest::prefillOutboundUpstreamQueue(uint32_t frame_count) { EXPECT_TRUE(upstream_request_->waitForData(*dispatcher_, 1)); // Make Envoy's writes into the upstream connection to return EAGAIN - writev_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); + write_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); auto buf = serializeFrames(Http2Frame::makePingFrame(), frame_count); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); auto* upstream = fake_upstreams_.front().get(); EXPECT_TRUE(upstream->rawWriteConnection(0, std::string(buf.begin(), buf.end()))); // Wait for pre-fill data to arrive to Envoy @@ -284,7 +284,7 @@ void Http2FloodMitigationTest::triggerListenerDrain() { TEST_P(Http2FloodMitigationTest, Ping) { setNetworkConnectionBufferSize(); beginSession(); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); floodServer(Http2Frame::makePingFrame(), "http2.outbound_control_flood", ControlFrameFloodLimit + 1); } @@ -292,7 +292,7 @@ TEST_P(Http2FloodMitigationTest, Ping) { TEST_P(Http2FloodMitigationTest, Settings) { setNetworkConnectionBufferSize(); beginSession(); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); floodServer(Http2Frame::makeEmptySettingsFrame(), "http2.outbound_control_flood", ControlFrameFloodLimit + 1); } @@ -325,7 +325,7 @@ TEST_P(Http2FloodMitigationTest, Data) { // 1000 DATA frames should trigger flood protection. // Simulate TCP push back on the Envoy's downstream network socket, so that outbound frames start // to accumulate in the transport socket buffer. - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); const auto request = Http2Frame::makeRequest(1, "host", "/test/long/url", {Http2Frame::Header("response_data_blocks", "1000"), @@ -565,7 +565,7 @@ TEST_P(Http2FloodMitigationTest, Trailers) { // 999 DATA frames and trailers should trigger flood protection. // Simulate TCP push back on the Envoy's downstream network socket, so that outbound frames start // to accumulate in the transport socket buffer. - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); static_cast(fake_upstreams_.front().get()) ->setResponseTrailers(std::make_unique( @@ -608,7 +608,7 @@ TEST_P(Http2FloodMitigationTest, WindowUpdateOnLowWatermarkFlood) { autonomous_allow_incomplete_streams_ = true; beginSession(); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); // pre-fill two away from overflow const auto request = Http2Frame::makePostRequest( @@ -673,7 +673,7 @@ TEST_P(Http2FloodMitigationTest, RST_STREAM) { // Simulate TCP push back on the Envoy's downstream network socket, so that outbound frames start // to accumulate in the transport socket buffer. - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); for (++stream_index; stream_index < ControlFrameFloodLimit + 2; ++stream_index) { request = @@ -949,7 +949,7 @@ TEST_P(Http2FloodMitigationTest, TooManyStreams) { // writing by the upstream server. In this case Envoy will not see upstream responses and will // keep client streams open, eventually maxing them out and causing client connection to be // closed. - writev_matcher_->setSourcePort(fake_upstreams_[0]->localAddress()->ip()->port()); + write_matcher_->setSourcePort(fake_upstreams_[0]->localAddress()->ip()->port()); // Exceed the number of streams allowed by the server. The server should stop reading from the // client. @@ -1446,9 +1446,9 @@ TEST_P(Http2FloodMitigationTest, RequestMetadata) { // Make Envoy's writes into the upstream connection to return EAGAIN, preventing proxying of the // METADATA frames - writev_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); + write_matcher_->setDestinationPort(fake_upstreams_[0]->localAddress()->ip()->port()); - writev_matcher_->setWritevReturnsEgain(); + write_matcher_->setWriteReturnsEgain(); // Send AllFrameFloodLimit + 1 number of METADATA frames from the downstream client to trigger the // outbound upstream flood when they are proxied. diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index a35ee6e29eb03..109e84dde5b66 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -39,6 +39,7 @@ #include "test/test_common/logging.h" #include "test/test_common/network_utility.h" #include "test/test_common/registry.h" +#include "test/test_common/threadsafe_singleton_injector.h" #include "absl/time/time.h" #include "gtest/gtest.h" @@ -3534,11 +3535,17 @@ TEST_P(DownstreamProtocolIntegrationTest, ContentLengthLargerThanPayload) { EXPECT_EQ(Http::StreamResetReason::RemoteReset, response->resetReason()); } +class NoUdpGso : public Api::OsSysCallsImpl { +public: + bool supportsUdpGso() const override { return false; } +}; + TEST_P(DownstreamProtocolIntegrationTest, HandleSocketFail) { + // Make sure for HTTP/3 Envoy will use sendmsg, so the write_matcher will work. + NoUdpGso reject_gso_; + TestThreadsafeSingletonInjector os_calls{&reject_gso_}; + ASSERT(!Api::OsSysCallsSingleton::get().supportsUdpGso()); SocketInterfaceSwap socket_swap; - if (downstreamProtocol() == Http::CodecType::HTTP3) { - return; - } initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -3547,12 +3554,23 @@ TEST_P(DownstreamProtocolIntegrationTest, HandleSocketFail) { // Makes us have Envoy's writes to downstream return EBADF Network::IoSocketError* ebadf = Network::IoSocketError::getIoSocketEbadfInstance(); - socket_swap.writev_matcher_->setSourcePort(lookupPort("http")); - socket_swap.writev_matcher_->setWritevOverride(ebadf); + socket_swap.write_matcher_->setSourcePort(lookupPort("http")); + socket_swap.write_matcher_->setWriteOverride(ebadf); + // TODO(danzh) set to true. upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, false); - ASSERT_TRUE(codec_client_->waitForDisconnect()); - socket_swap.writev_matcher_->setWritevOverride(nullptr); + if (downstreamProtocol() == Http::CodecType::HTTP3) { + // For HTTP/3 since the packets are black holed, there is no client side + // indication of connection close. Wait on Envoy stats instead. + test_server_->waitForCounterEq("http.config_test.downstream_rq_rx_reset", 1); + codec_client_->close(); + } else { + ASSERT_TRUE(codec_client_->waitForDisconnect()); + } + socket_swap.write_matcher_->setWriteOverride(nullptr); + // Shut down the server before os_calls goes out of scope to avoid syscalls + // during its removal. + test_server_.reset(); } } // namespace Envoy diff --git a/test/integration/socket_interface_swap.cc b/test/integration/socket_interface_swap.cc index 94d8abba9fbd1..f2b716127d48a 100644 --- a/test/integration/socket_interface_swap.cc +++ b/test/integration/socket_interface_swap.cc @@ -2,17 +2,18 @@ namespace Envoy { +void preserveIoError(Api::IoError*) {} + SocketInterfaceSwap::SocketInterfaceSwap() { Envoy::Network::SocketInterfaceSingleton::clear(); test_socket_interface_loader_ = std::make_unique( std::make_unique( - [writev_matcher = writev_matcher_](Envoy::Network::TestIoSocketHandle* io_handle, - const Buffer::RawSlice*, - uint64_t) -> absl::optional { - Network::IoSocketError* error_override = writev_matcher->returnOverride(io_handle); + [write_matcher = write_matcher_](Envoy::Network::TestIoSocketHandle* io_handle, + const Buffer::RawSlice*, + uint64_t) -> absl::optional { + Network::IoSocketError* error_override = write_matcher->returnOverride(io_handle); if (error_override) { - return Api::IoCallUint64Result( - 0, Api::IoErrorPtr(error_override, Network::IoSocketError::deleteIoError)); + return Api::IoCallUint64Result(0, Api::IoErrorPtr(error_override, preserveIoError)); } return absl::nullopt; })); diff --git a/test/integration/socket_interface_swap.h b/test/integration/socket_interface_swap.h index 820041a5d4eb0..676b8989453e7 100644 --- a/test/integration/socket_interface_swap.h +++ b/test/integration/socket_interface_swap.h @@ -12,12 +12,12 @@ namespace Envoy { class SocketInterfaceSwap { public: // Object of this class hold the state determining the IoHandle which - // should return EAGAIN from the `writev` call. + // should return the supplied return from the `writev` or `sendmsg` calls. struct IoHandleMatcher { Network::IoSocketError* returnOverride(Envoy::Network::TestIoSocketHandle* io_handle) { absl::MutexLock lock(&mutex_); if (error_ && (io_handle->localAddress()->ip()->port() == src_port_ || - io_handle->peerAddress()->ip()->port() == dst_port_)) { + (dst_port_ && io_handle->peerAddress()->ip()->port() == dst_port_))) { ASSERT(matched_iohandle_ == nullptr || matched_iohandle_ == io_handle, "Matched multiple io_handles, expected at most one to match."); matched_iohandle_ = io_handle; @@ -40,12 +40,12 @@ class SocketInterfaceSwap { dst_port_ = port; } - void setWritevReturnsEgain() { - setWritevOverride(Network::IoSocketError::getIoSocketEagainInstance()); + void setWriteReturnsEgain() { + setWriteOverride(Network::IoSocketError::getIoSocketEagainInstance()); } // The caller is responsible for memory management. - void setWritevOverride(Network::IoSocketError* error) { + void setWriteOverride(Network::IoSocketError* error) { absl::WriterMutexLock lock(&mutex_); ASSERT(src_port_ != 0 || dst_port_ != 0); error_ = error; @@ -70,7 +70,7 @@ class SocketInterfaceSwap { Envoy::Network::SocketInterface* const previous_socket_interface_{ Envoy::Network::SocketInterfaceSingleton::getExisting()}; - std::shared_ptr writev_matcher_{std::make_shared()}; + std::shared_ptr write_matcher_{std::make_shared()}; std::unique_ptr test_socket_interface_loader_; }; From 0e33d22bd4fd567269a7249b0b73cdc47877bece Mon Sep 17 00:00:00 2001 From: asraa Date: Thu, 2 Dec 2021 17:56:21 -0600 Subject: [PATCH 48/51] try fix for oss-fuzz build (#19164) Signed-off-by: Asra Ali --- bazel/repositories.bzl | 3 +++ bazel/rules_fuzzing.patch | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 bazel/rules_fuzzing.patch diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index e2173ae264a85..a7b187d19f22a 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1058,6 +1058,9 @@ def _rules_fuzzing(): repo_mapping = { "@fuzzing_py_deps": "@fuzzing_pip3", }, + # TODO(asraa): Try this fix for OSS-Fuzz build failure on tar command. + patch_args = ["-p1"], + patches = ["@envoy//bazel:rules_fuzzing.patch"], ) def _kafka_deps(): diff --git a/bazel/rules_fuzzing.patch b/bazel/rules_fuzzing.patch new file mode 100644 index 0000000000000..eca1b56e4d525 --- /dev/null +++ b/bazel/rules_fuzzing.patch @@ -0,0 +1,13 @@ +diff --git a/fuzzing/private/oss_fuzz/package.bzl b/fuzzing/private/oss_fuzz/package.bzl +index e5e9dc4..a3bb1b8 100644 +--- a/fuzzing/private/oss_fuzz/package.bzl ++++ b/fuzzing/private/oss_fuzz/package.bzl +@@ -71,7 +71,7 @@ def _oss_fuzz_package_impl(ctx): + if [[ -n "{options_path}" ]]; then + ln -s "$(pwd)/{options_path}" "$STAGING_DIR/{base_name}.options" + fi +- tar -chf "{output}" -C "$STAGING_DIR" . ++ tar -czhf "{output}" -C "$STAGING_DIR" . + """.format( + base_name = ctx.attr.base_name, + binary_path = binary_info.binary_file.path, \ No newline at end of file From e2e0a744a402b12fe4e5cf7539134977f707b4f3 Mon Sep 17 00:00:00 2001 From: Scott LaVigne <1406859+lavignes@users.noreply.github.com> Date: Thu, 2 Dec 2021 18:03:44 -0800 Subject: [PATCH 49/51] wasm: Fix potential segfault when reading filter_state (#17880) Other properties check for non-null `info` before attempting to read since it can be null. Signed-off-by: LaVigne, Scott --- source/extensions/common/wasm/context.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/extensions/common/wasm/context.cc b/source/extensions/common/wasm/context.cc index d3db5864dd47b..542ceb9e1b1db 100644 --- a/source/extensions/common/wasm/context.cc +++ b/source/extensions/common/wasm/context.cc @@ -565,9 +565,12 @@ Context::findValue(absl::string_view name, Protobuf::Arena* arena, bool last) co case PropertyToken::PLUGIN_VM_ID: return CelValue::CreateStringView(toAbslStringView(wasm()->vm_id())); case PropertyToken::FILTER_STATE: - return Protobuf::Arena::Create(arena, - info->filterState()) - ->Produce(arena); + if (info) { + return Protobuf::Arena::Create(arena, + info->filterState()) + ->Produce(arena); + } + break; } return {}; } From 3c27f5f8c21affc51983bedb603a5b8ca3009105 Mon Sep 17 00:00:00 2001 From: antonio Date: Thu, 2 Dec 2021 19:17:30 -0800 Subject: [PATCH 50/51] Simple benchmark to compare OwnedImpl::add to WatermarkBuffer::add (#19169) Also, benchmark to cover buffer account use. Signed-off-by: Antonio Vicente --- test/common/buffer/BUILD | 2 + test/common/buffer/buffer_speed_test.cc | 85 +++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/test/common/buffer/BUILD b/test/common/buffer/BUILD index 43cfdcf596084..1e17b41e4e8e6 100644 --- a/test/common/buffer/BUILD +++ b/test/common/buffer/BUILD @@ -112,6 +112,8 @@ envoy_cc_benchmark_binary( ], deps = [ "//source/common/buffer:buffer_lib", + "//source/common/buffer:watermark_buffer_lib", + "@envoy_api//envoy/config/overload/v3:pkg_cc_proto", ], ) diff --git a/test/common/buffer/buffer_speed_test.cc b/test/common/buffer/buffer_speed_test.cc index 458a92098aa00..d9ea2df1efb99 100644 --- a/test/common/buffer/buffer_speed_test.cc +++ b/test/common/buffer/buffer_speed_test.cc @@ -1,4 +1,8 @@ +#include "envoy/config/overload/v3/overload.pb.h" +#include "envoy/http/stream_reset_handler.h" + #include "source/common/buffer/buffer_impl.h" +#include "source/common/buffer/watermark_buffer.h" #include "source/common/common/assert.h" #include "absl/strings/string_view.h" @@ -8,6 +12,11 @@ namespace Envoy { static constexpr uint64_t MaxBufferLength = 1024 * 1024; +class FakeStreamResetHandler : public Http::StreamResetHandler { +public: + void resetStream(Http::StreamResetReason reason) override { UNREFERENCED_PARAMETER(reason); } +}; + // The fragment needs to be heap allocated in order to survive past the processing done in the inner // loop in the benchmarks below. Do not attempt to release the actual contents of the buffer. void deleteFragment(const void*, size_t, const Buffer::BufferFragmentImpl* self) { delete self; } @@ -24,6 +33,82 @@ static void bufferCreateEmpty(benchmark::State& state) { } BENCHMARK(bufferCreateEmpty); +// Test add performance of OwnedImpl vs WatermarkBuffer +static void bufferVsWatermarkBuffer(benchmark::State& state) { + const uint64_t length = state.range(0); + const uint64_t high_watermark = state.range(1); + const uint64_t step = state.range(2); + const bool use_watermark_buffer = (state.range(3) != 0); + const std::string data(step, 'a'); + + for (auto _ : state) { + UNREFERENCED_PARAMETER(_); + std::unique_ptr buffer; + if (use_watermark_buffer) { + buffer = std::make_unique([]() {}, []() {}, []() {}); + buffer->setWatermarks(high_watermark); + } else { + buffer = std::make_unique(); + } + + for (uint64_t idx = 0; idx < length; idx += step) { + buffer->add(data); + } + } +} +BENCHMARK(bufferVsWatermarkBuffer) + ->Args({1024, 0, 1, 0}) + ->Args({1024, 0, 1, 1}) + ->Args({1024, 1, 1, 1}) + ->Args({1024, 1024, 1, 1}) + ->Args({64 * 1024, 0, 1, 0}) + ->Args({64 * 1024, 0, 1, 1}) + ->Args({64 * 1024, 1, 1, 1}) + ->Args({64 * 1024, 64 * 1024, 1, 1}) + ->Args({64 * 1024, 0, 32, 0}) + ->Args({64 * 1024, 64 * 1024, 32, 1}) + ->Args({1024 * 1024, 0, 1, 0}) + ->Args({1024 * 1024, 0, 1, 1}) + ->Args({1024 * 1024, 1, 1, 1}) + ->Args({1024 * 1024, 1024 * 1024, 1, 1}) + ->Args({1024 * 1024, 0, 32, 0}) + ->Args({1024 * 1024, 1024 * 1024, 32, 1}); + +// Measure performance impact of enabling accounts. +static void bufferAccountUse(benchmark::State& state) { + const std::string data(state.range(0), 'a'); + uint64_t iters = state.range(1); + const bool enable_accounts = (state.range(2) != 0); + + auto config = envoy::config::overload::v3::BufferFactoryConfig(); + config.set_minimum_account_to_track_power_of_two(2); + Buffer::WatermarkBufferFactory buffer_factory(config); + FakeStreamResetHandler reset_handler; + auto account = buffer_factory.createAccount(reset_handler); + RELEASE_ASSERT(account != nullptr, ""); + + for (auto _ : state) { + UNREFERENCED_PARAMETER(_); + Buffer::OwnedImpl buffer; + if (enable_accounts) { + buffer.bindAccount(account); + } + for (uint64_t idx = 0; idx < iters; ++idx) { + buffer.add(data); + } + } + account->clearDownstream(); +} +BENCHMARK(bufferAccountUse) + ->Args({1, 1024 * 1024, 0}) + ->Args({1, 1024 * 1024, 1}) + ->Args({1024, 1024, 0}) + ->Args({1024, 1024, 1}) + ->Args({4 * 1024, 1024, 0}) + ->Args({4 * 1024, 1024, 1}) + ->Args({16 * 1024, 1024, 0}) + ->Args({16 * 1024, 1024, 1}); + // Test the creation of an OwnedImpl with varying amounts of content. static void bufferCreate(benchmark::State& state) { const std::string data(state.range(0), 'a'); From b44d31114d7a41037bc3205796ec190c8f9ad4ec Mon Sep 17 00:00:00 2001 From: Yao Zengzeng Date: Fri, 3 Dec 2021 17:13:07 +0800 Subject: [PATCH 51/51] doc: fix incorrect link (#19179) Signed-off-by: YaoZengzeng --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b6e47913f07f..20e8389e3ba6c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -197,7 +197,7 @@ Runtime features are set true by default by inclusion in There are four suggested options for testing new runtime features: 1. Create a per-test Runtime::LoaderSingleton as done in [DeprecatedFieldsTest.IndividualFieldDisallowedWithRuntimeOverride](https://github.com/envoyproxy/envoy/blob/main/test/common/protobuf/utility_test.cc) -2. Create a [parameterized test](https://github.com/google/googletest/blob/master/googletest/docs/advanced.md#how-to-write-value-parameterized-tests) +2. Create a [parameterized test](https://github.com/google/googletest/blob/master/docs/advanced.md#how-to-write-value-parameterized-tests) where the set up of the test sets the new runtime value explicitly to GetParam() as outlined in (1). 3. Set up integration tests with custom runtime defaults as documented in the