Skip to content
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion include/envoy/http/codec.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ const char MaxResponseHeadersCountOverrideKey[] =

class Stream;

enum class ErrorCode {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe call this GoAwayErrorCode? to make it clearer what kind of error codes are being modeled here

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 will update. Initially this enumerated all the error codes here, which are shared with RST_STREAM. But we've since undone that and there's already a separate Envoy-internal enum for stream resets.

NoError,
Other,
};

/**
* Stream encoder options specific to HTTP/1.
*/
Expand Down Expand Up @@ -324,7 +329,7 @@ class ConnectionCallbacks {
/**
* Fires when the remote indicates "go away." No new streams should be created.
*/
virtual void onGoAway() PURE;
virtual void onGoAway(ErrorCode error_code) PURE;
};

/**
Expand Down
4 changes: 2 additions & 2 deletions source/common/http/codec_client.h
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,9 @@ class CodecClient : Logger::Loggable<Logger::Id::client>,
Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher);

// Http::ConnectionCallbacks
void onGoAway() override {
void onGoAway(ErrorCode error_code) override {
if (codec_callbacks_) {
codec_callbacks_->onGoAway();
codec_callbacks_->onGoAway(error_code);
}
}

Expand Down
2 changes: 1 addition & 1 deletion source/common/http/conn_manager_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,7 @@ void ConnectionManagerImpl::doConnectionClose(
}
}

void ConnectionManagerImpl::onGoAway() {
void ConnectionManagerImpl::onGoAway(ErrorCode) {
// Currently we do nothing with remote go away frames. In the future we can decide to no longer
// push resources if applicable.
}
Expand Down
2 changes: 1 addition & 1 deletion source/common/http/conn_manager_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ class ConnectionManagerImpl : Logger::Loggable<Logger::Id::http>,
void initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) override;

// Http::ConnectionCallbacks
void onGoAway() override;
void onGoAway(ErrorCode error_code) override;

// Http::ServerConnectionCallbacks
RequestDecoder& newStream(ResponseEncoder& response_encoder,
Expand Down
20 changes: 14 additions & 6 deletions source/common/http/http2/codec_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,7 @@ ConnectionImpl::ConnectionImpl(Network::Connection& connection, CodecStats& stat
http2_options.max_inbound_priority_frames_per_stream().value()),
max_inbound_window_update_frames_per_data_frame_sent_(
http2_options.max_inbound_window_update_frames_per_data_frame_sent().value()),
dispatching_(false), raised_goaway_(false), pending_deferred_reset_(false) {}
dispatching_(false), pending_deferred_reset_(false) {}

ConnectionImpl::~ConnectionImpl() { nghttp2_session_del(session_); }

Expand Down Expand Up @@ -563,6 +563,16 @@ int ConnectionImpl::onBeforeFrameReceived(const nghttp2_frame_hd* hd) {
return 0;
}

ABSL_MUST_USE_RESULT
enum ErrorCode ngHttp2ErrorCodeToErrorCode(uint32_t code) noexcept {
switch (code) {
case NGHTTP2_NO_ERROR:
return ErrorCode::NoError;
default:
return ErrorCode::Other;
}
}

int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) {
ENVOY_CONN_LOG(trace, "recv frame type={}", connection_, static_cast<uint64_t>(frame->hd.type));

Expand All @@ -577,12 +587,10 @@ int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) {
}
}

// Only raise GOAWAY once, since we don't currently expose stream information. Shutdown
// notifications are the same as a normal GOAWAY.
if (frame->hd.type == NGHTTP2_GOAWAY && !raised_goaway_) {
// Shutdown notifications are the same as a normal GOAWAY.
if (frame->hd.type == NGHTTP2_GOAWAY) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if this is fine or if we want to track which goaway error codes we've already seen, and run the callbacks once per error code.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it expected to get multiple GOAWAY frames with different error codes? If yes, it seems bad only look at the first one, if no then we can probably keep the old logic.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is possible to receive GOAWAY frames with different error codes:

An endpoint MAY send multiple GOAWAY frames if circumstances change. For instance, an endpoint that sends GOAWAY with NO_ERROR during graceful shutdown could subsequently encounter a condition that requires immediate termination of the connection.

(from https://http2.github.io/http2-spec/#GOAWAY)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm in that case should the logic for failing the HC be that we see a NO_ERROR + no error GOAWAYs?

Also I'm not sure what the implications are of making onGoAway trigger multiple times per stream, maybe @mattklein123 or @alyssawilk has an idea? I'd be worried about invariants elsewhere relying on this being called at most once.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm in that case should the logic for failing the HC be that we see a NO_ERROR + no error GOAWAYs?

Hm, I don't quite follow, could you rephrase?

I'd be worried about invariants elsewhere relying on this being called at most once.

Agreed. It seems it was added in #103. I'm not sure if that was an optimization back then. I'm also not sure if it has become an expected behaviour (intentionally or not) since then.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  if (request_encoder_ && error_code == Http::GoAwayErrorCode::NoError) {
    received_no_error_goaway_ = true;
    return;
  }

if a GOAWAY error follows a NO_ERROR then received_no_error_goaway_ would still be true. I was thinking that the logic should reset this flag if it saw an error frame.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see. I don't expect that to change the behaviour as it stands:

  • If there isn't an active probe, the first (NO_ERROR) GOAWAY would've caused the connection to be closed (and we only set received_no_error_goaway_ = true; when there's an active probe).
  • If there is an active probe, the stream reset will end up resetting state (resetState() will be called by onResetStream()).
  • In general, I believe a connection close will end up resetting state because the onResetStream() callbacks will run (from here).

I admit this isn't obvious / does feel a bit brittle -- happy to add a comment and/or reset the flag anyways.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Friendly ping @mattklein123 @alyssawilk

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for the delay. My preference would be to not raise multiple goaways from the codec as I'm guessing there will be code that won't expect this and will break.

Can we just keep the previous behavior and not handle the case where multiple go away frames are received? We can potentially deal with this in a follow up if someone requests it?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the feedback - that seems reasonable to me. I've reverted that specific commit and added a TODO instead.

ASSERT(frame->hd.stream_id == 0);
raised_goaway_ = true;
callbacks().onGoAway();
callbacks().onGoAway(ngHttp2ErrorCodeToErrorCode(frame->goaway.error_code));
return 0;
}

Expand Down
1 change: 0 additions & 1 deletion source/common/http/http2/codec_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,6 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable<Log
void releaseOutboundControlFrame(const Buffer::OwnedBufferFragmentImpl* fragment);

bool dispatching_ : 1;
bool raised_goaway_ : 1;
bool pending_deferred_reset_ : 1;
};

Expand Down
2 changes: 1 addition & 1 deletion source/common/http/http2/conn_pool.cc
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ ConnPoolImpl::~ConnPoolImpl() { destructAllConnections(); }
ConnPoolImplBase::ActiveClientPtr ConnPoolImpl::instantiateActiveClient() {
return std::make_unique<ActiveClient>(*this);
}
void ConnPoolImpl::onGoAway(ActiveClient& client) {
void ConnPoolImpl::onGoAway(ActiveClient& client, Http::ErrorCode) {
ENVOY_CONN_LOG(debug, "remote goaway", *client.codec_client_);
host_->cluster().stats().upstream_cx_close_notify_.inc();
if (client.state_ != ActiveClient::State::DRAINING) {
Expand Down
4 changes: 2 additions & 2 deletions source/common/http/http2/conn_pool.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,14 @@ class ConnPoolImpl : public ConnPoolImplBase {
}

// Http::ConnectionCallbacks
void onGoAway() override { parent().onGoAway(*this); }
void onGoAway(Http::ErrorCode error_code) override { parent().onGoAway(*this, error_code); }

bool closed_with_active_rq_{};
};

uint64_t maxRequestsPerConnection();
void movePrimaryClientToDraining();
void onGoAway(ActiveClient& client);
void onGoAway(ActiveClient& client, Http::ErrorCode error_code);
void onStreamDestroy(ActiveClient& client);
void onStreamReset(ActiveClient& client, Http::StreamResetReason reason);

Expand Down
28 changes: 22 additions & 6 deletions source/common/upstream/health_checker_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -703,19 +703,22 @@ void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::onInterval() {
void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::onResetStream(Http::StreamResetReason,
absl::string_view) {
const bool expected_reset = expect_reset_;
const bool goaway = received_goaway_;
resetState();

if (expected_reset) {
// Stream reset was initiated by us (bogus gRPC response, timeout or cluster host is going
// away). In these cases health check failure has already been reported, so just return.
// away). In these cases health check failure has already been reported and a GOAWAY (if any)
// has already been handled, so just return.
return;
}

ENVOY_CONN_LOG(debug, "connection/stream error health_flags={}", *client_,
HostUtility::healthFlagsToString(*host_));

if (!parent_.reuse_connection_) {
// Stream reset was unexpected, so we haven't closed the connection yet.
if (goaway || !parent_.reuse_connection_) {
// Stream reset was unexpected, so we haven't closed the connection
// yet in response to a GOAWAY or due to disabled connection reuse.
client_->close();
}

Expand All @@ -727,9 +730,18 @@ void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::onResetStream(Http::St
handleFailure(envoy::data::core::v3::NETWORK);
}

void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::onGoAway() {
void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::onGoAway(Http::ErrorCode error_code) {
ENVOY_CONN_LOG(debug, "connection going away health_flags={}", *client_,
HostUtility::healthFlagsToString(*host_));
// If we have an active health check probe and receive a GOAWAY indicating
// graceful shutdown, allow the probe to complete before closing the connection.
// The connection will be closed when the active check completes or another
// terminal condition occurs, such as a timeout or stream reset.
if (request_encoder_ && error_code == Http::ErrorCode::NoError) {
received_goaway_ = true;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A one liner comment here about when this connection would eventually get closed would be nice. IIUC it's either on the next timeout or when the remote closes it?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll add a comment. It would be closed when the active check completes (onRpcComplete) or other conditions like onResetStream and onTimeout.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe rename this to make it clear that we received a non-error GOAWAY?

return;
}

// Even if we have active health check probe, fail it on GOAWAY and schedule new one.
if (request_encoder_) {
handleFailure(envoy::data::core::v3::NETWORK);
Expand Down Expand Up @@ -762,6 +774,9 @@ void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::onRpcComplete(
handleFailure(envoy::data::core::v3::ACTIVE);
}

// Read the value as we may call resetState() and clear it.
const bool goaway = received_goaway_;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Mind adding a comment here explaining why we have to read the value here? Its a bit less clear than on L705 since we're not unconditionally calling resetState


// |end_stream| will be false if we decided to stop healthcheck before HTTP stream has ended -
// invalid gRPC payload, unexpected message stream or wrong content-type.
if (end_stream) {
Expand All @@ -772,7 +787,7 @@ void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::onRpcComplete(
request_encoder_->getStream().resetStream(Http::StreamResetReason::LocalReset);
}

if (!parent_.reuse_connection_) {
if (!parent_.reuse_connection_ || goaway) {
client_->close();
}
}
Expand All @@ -782,13 +797,14 @@ void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::resetState() {
request_encoder_ = nullptr;
decoder_ = Grpc::Decoder();
health_check_response_.reset();
received_goaway_ = false;
}

void GrpcHealthCheckerImpl::GrpcActiveHealthCheckSession::onTimeout() {
ENVOY_CONN_LOG(debug, "connection/stream timeout health_flags={}", *client_,
HostUtility::healthFlagsToString(*host_));
expect_reset_ = true;
Comment thread
snowp marked this conversation as resolved.
if (!parent_.reuse_connection_) {
if (received_goaway_ || !parent_.reuse_connection_) {
client_->close();
} else {
request_encoder_->getStream().resetStream(Http::StreamResetReason::LocalReset);
Expand Down
5 changes: 3 additions & 2 deletions source/common/upstream/health_checker_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ class GrpcHealthCheckerImpl : public HealthCheckerImplBase {
void onBelowWriteBufferLowWatermark() override {}

void onEvent(Network::ConnectionEvent event);
void onGoAway();
void onGoAway(Http::ErrorCode error_code);

class ConnectionCallbackImpl : public Network::ConnectionCallbacks {
public:
Expand All @@ -341,7 +341,7 @@ class GrpcHealthCheckerImpl : public HealthCheckerImplBase {
public:
HttpConnectionCallbackImpl(GrpcActiveHealthCheckSession& parent) : parent_(parent) {}
// Http::ConnectionCallbacks
void onGoAway() override { parent_.onGoAway(); }
void onGoAway(Http::ErrorCode error_code) override { parent_.onGoAway(error_code); }

private:
GrpcActiveHealthCheckSession& parent_;
Expand All @@ -358,6 +358,7 @@ class GrpcHealthCheckerImpl : public HealthCheckerImplBase {
// e.g. remote reset. In this case healthcheck status has already been reported, only state
// cleanup is required.
bool expect_reset_ = false;
bool received_goaway_ = false;
};

virtual Http::CodecClientPtr createCodecClient(Upstream::Host::CreateConnectionData& data) PURE;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#include "extensions/quic_listeners/quiche/envoy_quic_client_session.h"

#include "extensions/quic_listeners/quiche/envoy_quic_utils.h"

namespace Envoy {
namespace Quic {

Expand Down Expand Up @@ -51,7 +53,7 @@ void EnvoyQuicClientSession::OnGoAway(const quic::QuicGoAwayFrame& frame) {
quic::QuicErrorCodeToString(frame.error_code), frame.reason_phrase);
quic::QuicSpdyClientSession::OnGoAway(frame);
if (http_connection_callbacks_ != nullptr) {
http_connection_callbacks_->onGoAway();
http_connection_callbacks_->onGoAway(quicErrorCodeToEnvoyErrorCode(frame.error_code));
}
}

Expand Down
9 changes: 9 additions & 0 deletions source/extensions/quic_listeners/quiche/envoy_quic_utils.cc
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,15 @@ Http::StreamResetReason quicErrorCodeToEnvoyResetReason(quic::QuicErrorCode erro
}
}

Http::ErrorCode quicErrorCodeToEnvoyErrorCode(quic::QuicErrorCode error) noexcept {
switch (error) {
case quic::QUIC_NO_ERROR:
return Http::ErrorCode::NoError;
default:
return Http::ErrorCode::Other;
}
}

Network::ConnectionSocketPtr
createConnectionSocket(Network::Address::InstanceConstSharedPtr& peer_addr,
Network::Address::InstanceConstSharedPtr& local_addr,
Expand Down
4 changes: 4 additions & 0 deletions source/extensions/quic_listeners/quiche/envoy_quic_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ Http::StreamResetReason quicRstErrorToEnvoyResetReason(quic::QuicRstStreamErrorC
// Called when underlying QUIC connection is closed either locally or by peer.
Http::StreamResetReason quicErrorCodeToEnvoyResetReason(quic::QuicErrorCode error);

// Called when a GOAWAY frame is received.
ABSL_MUST_USE_RESULT
Http::ErrorCode quicErrorCodeToEnvoyErrorCode(quic::QuicErrorCode error) noexcept;

// Create a connection socket instance and apply given socket options to the
// socket. IP_PKTINFO and SO_RXQ_OVFL is always set if supported.
Network::ConnectionSocketPtr
Expand Down
2 changes: 1 addition & 1 deletion test/common/http/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class CodecClientForTest : public Http::CodecClient {
destroy_cb_(this);
}
}
void raiseGoAway() { onGoAway(); }
void raiseGoAway(Http::ErrorCode error_code) { onGoAway(error_code); }
Event::Timer* idleTimer() { return idle_timer_.get(); }

DestroyCb destroy_cb_;
Expand Down
5 changes: 3 additions & 2 deletions test/common/http/http2/codec_impl_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -317,7 +317,8 @@ TEST_P(Http2CodecImplTest, ShutdownNotice) {
EXPECT_CALL(request_decoder_, decodeHeaders_(_, true));
request_encoder_->encodeHeaders(request_headers, true);

EXPECT_CALL(client_callbacks_, onGoAway());
// Called once from the shutdown notice and once from the goaway.
EXPECT_CALL(client_callbacks_, onGoAway(_)).Times(2);
server_->shutdownNotice();
server_->goAway();

Expand Down Expand Up @@ -1456,7 +1457,7 @@ TEST_P(Http2CodecImplTest, LargeRequestHeadersExceedPerHeaderLimit) {
request_headers.addCopy("big", long_string);

EXPECT_CALL(request_decoder_, decodeHeaders_(_, _)).Times(0);
EXPECT_CALL(client_callbacks_, onGoAway());
EXPECT_CALL(client_callbacks_, onGoAway(_));

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't completely follow why this test doesn't trigger onGoAway(_) to be called twice like the other one.

server_->shutdownNotice();
server_->goAway();
request_encoder_->encodeHeaders(request_headers, true);
Expand Down
2 changes: 1 addition & 1 deletion test/common/http/http2/conn_pool_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -971,7 +971,7 @@ TEST_F(Http2ConnPoolImplTest, GoAway) {
r1.inner_decoder_->decodeHeaders(
ResponseHeaderMapPtr{new TestResponseHeaderMapImpl{{":status", "200"}}}, true);

test_clients_[0].codec_client_->raiseGoAway();
test_clients_[0].codec_client_->raiseGoAway(Http::ErrorCode::NoError);

expectClientCreate();
ActiveTestRequest r2(*this, 1, false);
Expand Down
Loading