From 4df069e6f2102ade2c6d97bf19f9cea70cb26e95 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 26 Jul 2021 12:20:42 +0000 Subject: [PATCH 01/39] access logging: introduce critical ALS endpoint Signed-off-by: Shikugawa --- .../extensions/access_loggers/grpc/v3/BUILD | 1 + .../access_loggers/grpc/v3/als.proto | 22 +++- .../access_loggers/grpc/v4alpha/BUILD | 1 + .../access_loggers/grpc/v4alpha/als.proto | 22 +++- api/envoy/service/accesslog/v3/als.proto | 18 +++ api/envoy/service/accesslog/v4alpha/als.proto | 21 ++++ .../extensions/access_loggers/grpc/v3/BUILD | 1 + .../access_loggers/grpc/v3/als.proto | 22 +++- .../access_loggers/grpc/v4alpha/BUILD | 1 + .../access_loggers/grpc/v4alpha/als.proto | 22 +++- .../envoy/service/accesslog/v3/als.proto | 18 +++ .../envoy/service/accesslog/v4alpha/als.proto | 21 ++++ source/extensions/access_loggers/common/BUILD | 2 + .../common/grpc_access_logger.h | 107 ++++++++++++++++++ .../grpc/grpc_access_log_impl.cc | 20 ++++ .../grpc/grpc_access_log_impl.h | 4 + .../open_telemetry/grpc_access_log_impl.h | 4 + .../common/grpc_access_logger_test.cc | 24 ++++ 18 files changed, 327 insertions(+), 4 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/BUILD b/api/envoy/extensions/access_loggers/grpc/v3/BUILD index 1c1a6f6b44235..25c7a9c3aed13 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/BUILD +++ b/api/envoy/extensions/access_loggers/grpc/v3/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/accesslog/v3:pkg", "//envoy/config/core/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index fa0a9f0f820d5..d02fec9241816 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.access_loggers.grpc.v3; +import "envoy/config/accesslog/v3/accesslog.proto"; import "envoy/config/core/v3/config_source.proto"; import "envoy/config/core/v3/grpc_service.proto"; @@ -54,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 7] +// [#next-free-field: 9] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.accesslog.v2.CommonGrpcAccessLogConfig"; @@ -86,4 +87,23 @@ message CommonGrpcAccessLogConfig { // `. // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; + + // Define the log condition as a filter to ensure that the destination is reached. + // Logs that match the filter are not forwarded to the normal endpoint, + // but are sent to a special endpoint. This endpoint will return ACK/NACK as a response, + // and Envoy will perform buffering upon receiving a NACK. The buffered message is + // resend until it is reached. Failure to send here means that + // + // 1. when a NACK message corresponding to the message sent is received + // 2. when the ACK message corresponding to the sent message buffer could not be + // received until the message_ack_timeout is reached. + // + // It is recommended to set a strict filter because setting a loose filter here may cause + // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. + config.accesslog.v3.AccessLogFilter buffer_log_filter = 7 + [(validate.rules).enum = {defined_only: true}]; + + // The time to wait for an ACK message. If no ACK message is returned after this time, + // the message is considered undeliverable and the failed transmission is buffered again. + google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gt {}}]; } diff --git a/api/envoy/extensions/access_loggers/grpc/v4alpha/BUILD b/api/envoy/extensions/access_loggers/grpc/v4alpha/BUILD index 83758c9e0b82b..3293a5a6b1ed3 100644 --- a/api/envoy/extensions/access_loggers/grpc/v4alpha/BUILD +++ b/api/envoy/extensions/access_loggers/grpc/v4alpha/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/accesslog/v4alpha:pkg", "//envoy/config/core/v4alpha:pkg", "//envoy/extensions/access_loggers/grpc/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", diff --git a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 9e6fb1e48386e..9f9f258a0bebb 100644 --- a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.access_loggers.grpc.v4alpha; +import "envoy/config/accesslog/v4alpha/accesslog.proto"; import "envoy/config/core/v4alpha/config_source.proto"; import "envoy/config/core/v4alpha/grpc_service.proto"; @@ -54,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 7] +// [#next-free-field: 9] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.access_loggers.grpc.v3.CommonGrpcAccessLogConfig"; @@ -86,4 +87,23 @@ message CommonGrpcAccessLogConfig { // `. // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; + + // Define the log condition as a filter to ensure that the destination is reached. + // Logs that match the filter are not forwarded to the normal endpoint, + // but are sent to a special endpoint. This endpoint will return ACK/NACK as a response, + // and Envoy will perform buffering upon receiving a NACK. The buffered message is + // resend until it is reached. Failure to send here means that + // + // 1. when a NACK message corresponding to the message sent is received + // 2. when the ACK message corresponding to the sent message buffer could not be + // received until the message_ack_timeout is reached. + // + // It is recommended to set a strict filter because setting a loose filter here may cause + // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. + config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7 + [(validate.rules).enum = {defined_only: true}]; + + // The time to wait for an ACK message. If no ACK message is returned after this time, + // the message is considered undeliverable and the failed transmission is buffered again. + google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gt {}}]; } diff --git a/api/envoy/service/accesslog/v3/als.proto b/api/envoy/service/accesslog/v3/als.proto index 94a290ad4a325..be36ea3855727 100644 --- a/api/envoy/service/accesslog/v3/als.proto +++ b/api/envoy/service/accesslog/v3/als.proto @@ -27,6 +27,10 @@ service AccessLogService { // expectation that it might be lossy. rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } + + rpc BufferedCriticalAccessLogs(StreamAccessLogsMessage) + returns (BufferedCriticalAccessLogsResponse) { + } } // Empty response for the StreamAccessLogs API. Will never be sent. See below. @@ -35,6 +39,20 @@ message StreamAccessLogsResponse { "envoy.service.accesslog.v2.StreamAccessLogsResponse"; } +// Response received to identify undelivered or delivered messages in BufferedCriticalAccessLogs. +message BufferedCriticalAccessLogsResponse { + enum Status { + // Indicates that the message has been received. + ACK = 0; + + // Indicates that the message has not been received. + NACK = 1; + } + + // This field is used to indicate the arrival status. + Status status = 1; +} + // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream // access logs without ever expecting a response. message StreamAccessLogsMessage { diff --git a/api/envoy/service/accesslog/v4alpha/als.proto b/api/envoy/service/accesslog/v4alpha/als.proto index ab0ba0e15213e..caa5292826e10 100644 --- a/api/envoy/service/accesslog/v4alpha/als.proto +++ b/api/envoy/service/accesslog/v4alpha/als.proto @@ -27,6 +27,10 @@ service AccessLogService { // expectation that it might be lossy. rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } + + rpc BufferedCriticalAccessLogs(StreamAccessLogsMessage) + returns (BufferedCriticalAccessLogsResponse) { + } } // Empty response for the StreamAccessLogs API. Will never be sent. See below. @@ -35,6 +39,23 @@ message StreamAccessLogsResponse { "envoy.service.accesslog.v3.StreamAccessLogsResponse"; } +// Response received to identify undelivered or delivered messages in BufferedCriticalAccessLogs. +message BufferedCriticalAccessLogsResponse { + option (udpa.annotations.versioning).previous_message_type = + "envoy.service.accesslog.v3.BufferedCriticalAccessLogsResponse"; + + enum Status { + // Indicates that the message has been received. + ACK = 0; + + // Indicates that the message has not been received. + NACK = 1; + } + + // This field is used to indicate the arrival status. + Status status = 1; +} + // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream // access logs without ever expecting a response. message StreamAccessLogsMessage { diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD index 1c1a6f6b44235..25c7a9c3aed13 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/accesslog/v3:pkg", "//envoy/config/core/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index fa0a9f0f820d5..d02fec9241816 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.access_loggers.grpc.v3; +import "envoy/config/accesslog/v3/accesslog.proto"; import "envoy/config/core/v3/config_source.proto"; import "envoy/config/core/v3/grpc_service.proto"; @@ -54,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 7] +// [#next-free-field: 9] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.accesslog.v2.CommonGrpcAccessLogConfig"; @@ -86,4 +87,23 @@ message CommonGrpcAccessLogConfig { // `. // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; + + // Define the log condition as a filter to ensure that the destination is reached. + // Logs that match the filter are not forwarded to the normal endpoint, + // but are sent to a special endpoint. This endpoint will return ACK/NACK as a response, + // and Envoy will perform buffering upon receiving a NACK. The buffered message is + // resend until it is reached. Failure to send here means that + // + // 1. when a NACK message corresponding to the message sent is received + // 2. when the ACK message corresponding to the sent message buffer could not be + // received until the message_ack_timeout is reached. + // + // It is recommended to set a strict filter because setting a loose filter here may cause + // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. + config.accesslog.v3.AccessLogFilter buffer_log_filter = 7 + [(validate.rules).enum = {defined_only: true}]; + + // The time to wait for an ACK message. If no ACK message is returned after this time, + // the message is considered undeliverable and the failed transmission is buffered again. + google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gt {}}]; } diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/BUILD b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/BUILD index 83758c9e0b82b..3293a5a6b1ed3 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/BUILD +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/accesslog/v4alpha:pkg", "//envoy/config/core/v4alpha:pkg", "//envoy/extensions/access_loggers/grpc/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 9e6fb1e48386e..9f9f258a0bebb 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.extensions.access_loggers.grpc.v4alpha; +import "envoy/config/accesslog/v4alpha/accesslog.proto"; import "envoy/config/core/v4alpha/config_source.proto"; import "envoy/config/core/v4alpha/grpc_service.proto"; @@ -54,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 7] +// [#next-free-field: 9] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.access_loggers.grpc.v3.CommonGrpcAccessLogConfig"; @@ -86,4 +87,23 @@ message CommonGrpcAccessLogConfig { // `. // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; + + // Define the log condition as a filter to ensure that the destination is reached. + // Logs that match the filter are not forwarded to the normal endpoint, + // but are sent to a special endpoint. This endpoint will return ACK/NACK as a response, + // and Envoy will perform buffering upon receiving a NACK. The buffered message is + // resend until it is reached. Failure to send here means that + // + // 1. when a NACK message corresponding to the message sent is received + // 2. when the ACK message corresponding to the sent message buffer could not be + // received until the message_ack_timeout is reached. + // + // It is recommended to set a strict filter because setting a loose filter here may cause + // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. + config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7 + [(validate.rules).enum = {defined_only: true}]; + + // The time to wait for an ACK message. If no ACK message is returned after this time, + // the message is considered undeliverable and the failed transmission is buffered again. + google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gt {}}]; } diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto index 94a290ad4a325..be36ea3855727 100644 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v3/als.proto @@ -27,6 +27,10 @@ service AccessLogService { // expectation that it might be lossy. rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } + + rpc BufferedCriticalAccessLogs(StreamAccessLogsMessage) + returns (BufferedCriticalAccessLogsResponse) { + } } // Empty response for the StreamAccessLogs API. Will never be sent. See below. @@ -35,6 +39,20 @@ message StreamAccessLogsResponse { "envoy.service.accesslog.v2.StreamAccessLogsResponse"; } +// Response received to identify undelivered or delivered messages in BufferedCriticalAccessLogs. +message BufferedCriticalAccessLogsResponse { + enum Status { + // Indicates that the message has been received. + ACK = 0; + + // Indicates that the message has not been received. + NACK = 1; + } + + // This field is used to indicate the arrival status. + Status status = 1; +} + // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream // access logs without ever expecting a response. message StreamAccessLogsMessage { diff --git a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto index ab0ba0e15213e..caa5292826e10 100644 --- a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto @@ -27,6 +27,10 @@ service AccessLogService { // expectation that it might be lossy. rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } + + rpc BufferedCriticalAccessLogs(StreamAccessLogsMessage) + returns (BufferedCriticalAccessLogsResponse) { + } } // Empty response for the StreamAccessLogs API. Will never be sent. See below. @@ -35,6 +39,23 @@ message StreamAccessLogsResponse { "envoy.service.accesslog.v3.StreamAccessLogsResponse"; } +// Response received to identify undelivered or delivered messages in BufferedCriticalAccessLogs. +message BufferedCriticalAccessLogsResponse { + option (udpa.annotations.versioning).previous_message_type = + "envoy.service.accesslog.v3.BufferedCriticalAccessLogsResponse"; + + enum Status { + // Indicates that the message has been received. + ACK = 0; + + // Indicates that the message has not been received. + NACK = 1; + } + + // This field is used to indicate the arrival status. + Status status = 1; +} + // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream // access logs without ever expecting a response. message StreamAccessLogsMessage { diff --git a/source/extensions/access_loggers/common/BUILD b/source/extensions/access_loggers/common/BUILD index d7968b6e0560c..2033bc2dd2cd1 100644 --- a/source/extensions/access_loggers/common/BUILD +++ b/source/extensions/access_loggers/common/BUILD @@ -59,5 +59,7 @@ envoy_cc_library( "//source/common/protobuf:utility_lib", "@com_google_absl//absl/types:optional", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/access_loggers/grpc/v3:pkg_cc_proto", + "@envoy_api//envoy/service/accesslog/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index 2e008a9c1522f..d4906720ba59c 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -1,7 +1,11 @@ #pragma once #include +#include +// TODO(shikugagwa): extensions should not be placed here +#include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" +#include "envoy/service/accesslog/v3/als.pb.h" #include "envoy/config/core/v3/config_source.pb.h" #include "envoy/event/dispatcher.h" #include "envoy/grpc/async_client_manager.h" @@ -141,6 +145,84 @@ template class GrpcAccessLogClient { const absl::optional transport_api_version_; }; +template class FatalAccessLoggerGrpcClient { +public: + using ResponseType = envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse; + + FatalAccessLoggerGrpcClient(const Grpc::RawAsyncClientSharedPtr& client, + const Protobuf::MethodDescriptor& method) + : FatalAccessLoggerGrpcClient(client, method, absl::nullopt) {} + FatalAccessLoggerGrpcClient( + const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& method, + absl::optional transport_api_version) + : client_(client), service_method_(method), transport_api_version_(transport_api_version) {} + + void flush(const RequestType& message) { + active_stream_ = std::make_unique(*this, message); + active_stream_->stream_ = + client_.start(service_method_, *active_stream_, Http::AsyncClient::StreamOptions()); + + if (active_stream_->stream_ == nullptr || + active_stream_->stream_.isAboveWriteBufferHighWatermark()) { + active_stream_.reset(); + return; + } + + if (transport_api_version_.has_value()) { + active_stream_->stream_->sendMessage(message, transport_api_version_.value(), false); + } else { + active_stream_->stream_->sendMessage(message, false); + } + + // TODO(shikugawa): stream must be ended with response withing specified timeout, + // that must be defined to setup duration after last message transported. + } + +private: + struct ActiveStream : public Grpc::AsyncStreamCallbacks { + ActiveStream(FatalAccessLoggerGrpcClient& parent, + const RequestType& request_message) + : parent_(parent), request_message_(request_message) {} + + // Grpc::AsyncStreamCallbacks + void onCreateInitialMetadata(Http::RequestHeaderMap&) override {} + void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&&) override {} + void onReceiveMessage(std::unique_ptr&& message) override { + switch (message->status()) { + case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK: + break; + case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK: + // TODO(shikugawa): After many NACK occurred, Buffer will be overwhelmed. + // To resolve this, Buffer should have size limit. + // TODO(shilugawa): Buffer failed message. + break; + default: + return; + } + } + void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {} + void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override { + ASSERT(parent_.active_stream_ != nullptr); + if (parent_.active_stream_->stream_ != nullptr) { + parent_.active_stream_.reset(); + } + } + + // TODO(shikugawa): define buffer. + FatalAccessLoggerGrpcClient& parent_; + Grpc::AsyncStream stream_; + const RequestType& request_message_; + }; + + friend ActiveStream; + + // absl::flat_hash_map> buffer_; + std::unique_ptr active_stream_; + Grpc::AsyncClient client_; + const Protobuf::MethodDescriptor& service_method_; + const absl::optional transport_api_version_; +}; + } // namespace Detail /** @@ -182,6 +264,10 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerFindMethodByName( + "envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs"), + transport_api_version), buffer_flush_interval_msec_(buffer_flush_interval_msec), flush_timer_(dispatcher.createTimer([this]() { flush(); @@ -193,6 +279,11 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger= max_buffer_size_bytes_) { @@ -213,13 +309,19 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger client_; + Detail::FatalAccessLoggerGrpcClient fatal_client_; LogRequest message_; + LogRequest fatal_message_; private: virtual bool isEmpty() PURE; virtual void initMessage() PURE; virtual void addEntry(HttpLogProto&& entry) PURE; virtual void addEntry(TcpLogProto&& entry) PURE; + virtual void addFatalEntry(HttpLogProto&& entry) PURE; + virtual void addFatalEntry(TcpLogProto&& entry) PURE; + virtual bool shouldBuffer(const HttpLogProto& entry) PURE; + virtual bool shouldBuffer(const TcpLogProto& entry) PURE; virtual void clearMessage() { message_.Clear(); } void flush() { @@ -237,6 +339,10 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggermutable_log_entry()->Add(std::move(entry)); } +void GrpcAccessLoggerImpl::addFatalEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { + fatal_message_.mutable_http_logs()->mutable_log_entry()->Add(std::move(entry)); +} + +void GrpcAccessLoggerImpl::addFatalEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) { + fatal_message_.mutable_tcp_logs()->mutable_log_entry()->Add(std::move(entry)); +} + +bool GrpcAccessLoggerImpl::shouldBuffer(const envoy::data::accesslog::v3::HTTPAccessLogEntry&) { + // TODO(shikugawa): To buffer message it determined as critical message, + // by the trigger of configured thresholds. e.g. matched specific filter + // which describes https log message has >= 500 status code. + return true; +} + +bool GrpcAccessLoggerImpl::shouldBuffer(const envoy::data::accesslog::v3::TCPAccessLogEntry&) { + // TODO(shikugawa): Not supported for TCP message. + return false; +} + bool GrpcAccessLoggerImpl::isEmpty() { return !message_.has_http_logs() && !message_.has_tcp_logs(); } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 07bf8400fdc1c..9587d925e31b9 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -33,6 +33,10 @@ class GrpcAccessLoggerImpl // Extensions::AccessLoggers::GrpcCommon::GrpcAccessLogger void addEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) override; void addEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) override; + void addFatalEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) override; + void addFatalEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) override; + bool shouldBuffer(const envoy::data::accesslog::v3::HTTPAccessLogEntry& entry) override; + bool shouldBuffer(const envoy::data::accesslog::v3::TCPAccessLogEntry& entry) override; bool isEmpty() override; void initMessage() override; diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h index 52e61e134ee3b..2b7024d102046 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h @@ -47,6 +47,10 @@ class GrpcAccessLoggerImpl void addEntry(opentelemetry::proto::logs::v1::LogRecord&& entry) override; // Non used addEntry method (the above is used for both TCP and HTTP). void addEntry(ProtobufWkt::Empty&& entry) override { (void)entry; }; + void addFatalEntry(opentelemetry::proto::logs::v1::LogRecord&&) override {} + void addFatalEntry(ProtobufWkt::Empty&&) override{}; + bool shouldBuffer(const opentelemetry::proto::logs::v1::LogRecord&) override { return false; } + bool shouldBuffer(const ProtobufWkt::Empty&) override { return false; } bool isEmpty() override; void initMessage() override; void clearMessage() override; diff --git a/test/extensions/access_loggers/common/grpc_access_logger_test.cc b/test/extensions/access_loggers/common/grpc_access_logger_test.cc index 57f5f2278624a..bf47a0e6c6cab 100644 --- a/test/extensions/access_loggers/common/grpc_access_logger_test.cc +++ b/test/extensions/access_loggers/common/grpc_access_logger_test.cc @@ -75,6 +75,16 @@ class MockGrpcAccessLoggerImpl 1); } + void mockAddFatalEntry(const std::string& key) { + if (!fatal_message_.fields().contains(key)) { + ProtobufWkt::Value default_value; + default_value.set_number_value(0); + fatal_message_.mutable_fields()->insert({key, default_value}); + } + fatal_message_.mutable_fields()->at(key).set_number_value( + fatal_message_.fields().at(key).number_value() + 1); + } + // Extensions::AccessLoggers::GrpcCommon::GrpcAccessLogger // For testing purposes, we don't really care how each of these virtual methods is implemented, as // it's up to each logger implementation. We test whether they were called in the regular flow of @@ -90,6 +100,18 @@ class MockGrpcAccessLoggerImpl mockAddEntry(MOCK_TCP_LOG_FIELD_NAME); } + void addFatalEntry(ProtobufWkt::Struct&& entry) override { + (void)entry; + mockAddFatalEntry(MOCK_HTTP_LOG_FIELD_NAME); + } + + void addFatalEntry(ProtobufWkt::Empty&& entry) override { + (void)entry; + mockAddFatalEntry(MOCK_TCP_LOG_FIELD_NAME); + } + bool shouldBuffer(const ProtobufWkt::Struct&) override { return http_should_buffer_; } + bool shouldBuffer(const ProtobufWkt::Empty&) override { return tcp_should_buffer_; } + bool isEmpty() override { return message_.fields().empty(); } void initMessage() override { ++num_inits_; } @@ -99,6 +121,8 @@ class MockGrpcAccessLoggerImpl num_clears_++; } + bool tcp_should_buffer_ = false; + bool http_should_buffer_ = false; int num_inits_ = 0; int num_clears_ = 0; }; From daa2f41954b9956a644dde287f5430db979de702 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 28 Jul 2021 13:13:36 +0000 Subject: [PATCH 02/39] wip Signed-off-by: Shikugawa --- .../access_loggers/grpc/v3/als.proto | 6 +- .../access_loggers/grpc/v4alpha/als.proto | 6 +- api/envoy/service/accesslog/v3/als.proto | 7 +- api/envoy/service/accesslog/v4alpha/als.proto | 7 +- .../access_loggers/grpc/v3/als.proto | 6 +- .../access_loggers/grpc/v4alpha/als.proto | 6 +- .../envoy/service/accesslog/v3/als.proto | 7 +- .../envoy/service/accesslog/v4alpha/als.proto | 7 +- .../common/grpc_access_logger.h | 150 ++++++++++++------ .../grpc/grpc_access_log_impl.cc | 30 ++-- .../grpc/grpc_access_log_impl.h | 4 +- .../access_loggers/grpc/http_config.cc | 2 +- .../grpc/http_grpc_access_log_impl.cc | 18 ++- .../grpc/http_grpc_access_log_impl.h | 3 +- .../grpc/tcp_grpc_access_log_impl.cc | 3 +- .../open_telemetry/access_log_impl.cc | 2 +- .../open_telemetry/grpc_access_log_impl.cc | 5 + .../open_telemetry/grpc_access_log_impl.h | 4 +- .../common/grpc_access_logger_test.cc | 32 ++-- .../grpc/grpc_access_log_impl_test.cc | 6 +- .../grpc/http_grpc_access_log_impl_test.cc | 15 +- .../open_telemetry/access_log_impl_test.cc | 9 +- .../grpc_access_log_impl_test.cc | 6 +- 23 files changed, 213 insertions(+), 128 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index d02fec9241816..e9dda14622265 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -100,10 +100,10 @@ message CommonGrpcAccessLogConfig { // // It is recommended to set a strict filter because setting a loose filter here may cause // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. - config.accesslog.v3.AccessLogFilter buffer_log_filter = 7 - [(validate.rules).enum = {defined_only: true}]; + config.accesslog.v3.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gt {}}]; + google.protobuf.Duration message_ack_timeout = 8 + [(validate.rules).duration = {gte {nanos: 1000000}}]; } diff --git a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 9f9f258a0bebb..58a0c937ee1a5 100644 --- a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -100,10 +100,10 @@ message CommonGrpcAccessLogConfig { // // It is recommended to set a strict filter because setting a loose filter here may cause // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. - config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7 - [(validate.rules).enum = {defined_only: true}]; + config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gt {}}]; + google.protobuf.Duration message_ack_timeout = 8 + [(validate.rules).duration = {gte {nanos: 1000000}}]; } diff --git a/api/envoy/service/accesslog/v3/als.proto b/api/envoy/service/accesslog/v3/als.proto index be36ea3855727..f6d8a5531ad81 100644 --- a/api/envoy/service/accesslog/v3/als.proto +++ b/api/envoy/service/accesslog/v3/als.proto @@ -28,8 +28,8 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - rpc BufferedCriticalAccessLogs(StreamAccessLogsMessage) - returns (BufferedCriticalAccessLogsResponse) { + rpc BufferedCriticalAccessLogs(stream StreamAccessLogsMessage) + returns (stream BufferedCriticalAccessLogsResponse) { } } @@ -51,6 +51,9 @@ message BufferedCriticalAccessLogsResponse { // This field is used to indicate the arrival status. Status status = 1; + + // Message ID that identifies a message. + uint32 id = 2; } // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream diff --git a/api/envoy/service/accesslog/v4alpha/als.proto b/api/envoy/service/accesslog/v4alpha/als.proto index caa5292826e10..cf91576a86306 100644 --- a/api/envoy/service/accesslog/v4alpha/als.proto +++ b/api/envoy/service/accesslog/v4alpha/als.proto @@ -28,8 +28,8 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - rpc BufferedCriticalAccessLogs(StreamAccessLogsMessage) - returns (BufferedCriticalAccessLogsResponse) { + rpc BufferedCriticalAccessLogs(stream StreamAccessLogsMessage) + returns (stream BufferedCriticalAccessLogsResponse) { } } @@ -54,6 +54,9 @@ message BufferedCriticalAccessLogsResponse { // This field is used to indicate the arrival status. Status status = 1; + + // Message ID that identifies a message. + uint32 id = 2; } // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index d02fec9241816..e9dda14622265 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -100,10 +100,10 @@ message CommonGrpcAccessLogConfig { // // It is recommended to set a strict filter because setting a loose filter here may cause // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. - config.accesslog.v3.AccessLogFilter buffer_log_filter = 7 - [(validate.rules).enum = {defined_only: true}]; + config.accesslog.v3.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gt {}}]; + google.protobuf.Duration message_ack_timeout = 8 + [(validate.rules).duration = {gte {nanos: 1000000}}]; } diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 9f9f258a0bebb..58a0c937ee1a5 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -100,10 +100,10 @@ message CommonGrpcAccessLogConfig { // // It is recommended to set a strict filter because setting a loose filter here may cause // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. - config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7 - [(validate.rules).enum = {defined_only: true}]; + config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gt {}}]; + google.protobuf.Duration message_ack_timeout = 8 + [(validate.rules).duration = {gte {nanos: 1000000}}]; } diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto index be36ea3855727..f6d8a5531ad81 100644 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v3/als.proto @@ -28,8 +28,8 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - rpc BufferedCriticalAccessLogs(StreamAccessLogsMessage) - returns (BufferedCriticalAccessLogsResponse) { + rpc BufferedCriticalAccessLogs(stream StreamAccessLogsMessage) + returns (stream BufferedCriticalAccessLogsResponse) { } } @@ -51,6 +51,9 @@ message BufferedCriticalAccessLogsResponse { // This field is used to indicate the arrival status. Status status = 1; + + // Message ID that identifies a message. + uint32 id = 2; } // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream diff --git a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto index caa5292826e10..cf91576a86306 100644 --- a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto @@ -28,8 +28,8 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - rpc BufferedCriticalAccessLogs(StreamAccessLogsMessage) - returns (BufferedCriticalAccessLogsResponse) { + rpc BufferedCriticalAccessLogs(stream StreamAccessLogsMessage) + returns (stream BufferedCriticalAccessLogsResponse) { } } @@ -54,6 +54,9 @@ message BufferedCriticalAccessLogsResponse { // This field is used to indicate the arrival status. Status status = 1; + + // Message ID that identifies a message. + uint32 id = 2; } // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index d4906720ba59c..ef062996ccd1c 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -13,6 +15,7 @@ #include "envoy/stats/scope.h" #include "envoy/thread_local/thread_local.h" +#include "source/common/access_log/access_log_impl.h" #include "source/common/common/assert.h" #include "source/common/grpc/typed_async_client.h" #include "source/common/protobuf/utility.h" @@ -48,14 +51,16 @@ template class GrpcAccessLogger { /** * Log http access entry. * @param entry supplies the access log to send. + * @param is_critical determine whether log entry is critical or not. */ - virtual void log(HttpLogProto&& entry) PURE; + virtual void log(HttpLogProto&& entry, bool is_critical) PURE; /** * Log tcp access entry. * @param entry supplies the access log to send. + * @param is_critical determine whether log entry is critical or not. */ - virtual void log(TcpLogProto&& entry) PURE; + virtual void log(TcpLogProto&& entry, bool is_critical) PURE; }; /** @@ -157,47 +162,38 @@ template class FatalAccessLoggerGrpcClient { absl::optional transport_api_version) : client_(client), service_method_(method), transport_api_version_(transport_api_version) {} - void flush(const RequestType& message) { - active_stream_ = std::make_unique(*this, message); - active_stream_->stream_ = - client_.start(service_method_, *active_stream_, Http::AsyncClient::StreamOptions()); - - if (active_stream_->stream_ == nullptr || - active_stream_->stream_.isAboveWriteBufferHighWatermark()) { - active_stream_.reset(); - return; - } - - if (transport_api_version_.has_value()) { - active_stream_->stream_->sendMessage(message, transport_api_version_.value(), false); - } else { - active_stream_->stream_->sendMessage(message, false); - } + struct BufferedMessage { + enum class State { + Initial, + Pending, + }; - // TODO(shikugawa): stream must be ended with response withing specified timeout, - // that must be defined to setup duration after last message transported. - } + State state_{State::Initial}; + ResponseType& message_; + }; -private: struct ActiveStream : public Grpc::AsyncStreamCallbacks { - ActiveStream(FatalAccessLoggerGrpcClient& parent, - const RequestType& request_message) - : parent_(parent), request_message_(request_message) {} + ActiveStream(FatalAccessLoggerGrpcClient& parent) : parent_(parent) {} // Grpc::AsyncStreamCallbacks void onCreateInitialMetadata(Http::RequestHeaderMap&) override {} void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&&) override {} void onReceiveMessage(std::unique_ptr&& message) override { - switch (message->status()) { - case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK: - break; - case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK: - // TODO(shikugawa): After many NACK occurred, Buffer will be overwhelmed. - // To resolve this, Buffer should have size limit. - // TODO(shilugawa): Buffer failed message. - break; - default: - return; + const auto& id = message->id(); + + if (parent_.buffered_messages_.find(id) != parent_.buffered_messages_.end()) { + switch (message->status()) { + case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK: + parent_.buffered_messages_.erase(id); + break; + case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK: + parent_.buffered_messages_.at(id).state_ = BufferedMessage::State::Initial; + // TODO(shikugawa): After many NACK occurred, Buffer will be overwhelmed. + // To resolve this, Buffer should have size limit. + break; + default: + return; + } } } void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {} @@ -208,15 +204,48 @@ template class FatalAccessLoggerGrpcClient { } } - // TODO(shikugawa): define buffer. FatalAccessLoggerGrpcClient& parent_; Grpc::AsyncStream stream_; - const RequestType& request_message_; }; + void flush() { + if (!active_stream_) { + active_stream_ = std::make_unique(*this); + } + + if (active_stream_->stream_ == nullptr) { + active_stream_->stream_ = + client_.start(service_method_, *active_stream_, Http::AsyncClient::StreamOptions()); + } + + if (active_stream_->stream_ == nullptr || + active_stream_->stream_.isAboveWriteBufferHighWatermark()) { + active_stream_.reset(); + return; + } + + for (auto&& buffered_message : buffered_messages_) { + uint32_t id = buffered_message.first; + buffered_message.second.message_.set_id(id); + buffered_message.second.state_ = BufferedMessage::State::Pending; + + // TODO(shikugawa): stream must be ended with response withing specified timeout, + // that must be defined to setup duration after last message transported. + if (transport_api_version_.has_value()) { + active_stream_->stream_->sendMessage(buffered_message.second.message_, + transport_api_version_.value(), false); + } else { + active_stream_->stream_->sendMessage(buffered_message.second.message_, false); + } + } + } + + bool isStreamStarted() { return active_stream_ != nullptr && active_stream_->stream_ != nullptr; } + +private: friend ActiveStream; - // absl::flat_hash_map> buffer_; + absl::flat_hash_map buffered_messages_; std::unique_ptr active_stream_; Grpc::AsyncClient client_; const Protobuf::MethodDescriptor& service_method_; @@ -264,13 +293,10 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerFindMethodByName( - "envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs"), - transport_api_version), buffer_flush_interval_msec_(buffer_flush_interval_msec), flush_timer_(dispatcher.createTimer([this]() { flush(); + flushFatal(); flush_timer_->enableTimer(buffer_flush_interval_msec_); })), max_buffer_size_bytes_(max_buffer_size_bytes), @@ -278,8 +304,14 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerenableTimer(buffer_flush_interval_msec_); } - void log(HttpLogProto&& entry) { - if (enable_fatal_log_ && shouldBuffer(entry)) { + void log(HttpLogProto&& entry, bool is_critical) { + // TODO(shikugawa): If the log is critical, it will be stored in a message for critical log, but + // for gRPC logger (e.g. OpenTelemetry) that does not support Critical Logging Endpoint, the + // critical message will not be flushed forever In the case of a gRPC logger (e.g. + // OpenTelemetry) that does not support critical logging endpoint, the critical message will not + // be flushed forever. This can lead to a continual memory drain. Messages for critical logs may + // need to be flushed at regular intervals in such cases. + if (is_critical) { addFatalEntry(std::move(entry)); return; } @@ -294,8 +326,8 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger>( + client, service_method, transport_api_version); + } + Detail::GrpcAccessLogClient client_; - Detail::FatalAccessLoggerGrpcClient fatal_client_; + std::unique_ptr> fatal_client_; LogRequest message_; LogRequest fatal_message_; private: virtual bool isEmpty() PURE; + virtual bool isFatalEmpty() PURE; virtual void initMessage() PURE; + virtual void initFatalMessage() PURE; virtual void addEntry(HttpLogProto&& entry) PURE; virtual void addEntry(TcpLogProto&& entry) PURE; virtual void addFatalEntry(HttpLogProto&& entry) PURE; virtual void addFatalEntry(TcpLogProto&& entry) PURE; - virtual bool shouldBuffer(const HttpLogProto& entry) PURE; - virtual bool shouldBuffer(const TcpLogProto& entry) PURE; virtual void clearMessage() { message_.Clear(); } void flush() { @@ -339,10 +378,18 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerisStreamStarted()) { + initFatalMessage(); + } + + fatal_client_->flush(); + // TODO(shikugawa): consider message and buffer lifecycle. } bool canLogMore() { @@ -364,7 +411,6 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerFindMethodByName( + "envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs"), + transport_api_version); +} void GrpcAccessLoggerImpl::addEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { message_.mutable_http_logs()->mutable_log_entry()->Add(std::move(entry)); @@ -45,28 +51,26 @@ void GrpcAccessLoggerImpl::addFatalEntry(envoy::data::accesslog::v3::TCPAccessLo fatal_message_.mutable_tcp_logs()->mutable_log_entry()->Add(std::move(entry)); } -bool GrpcAccessLoggerImpl::shouldBuffer(const envoy::data::accesslog::v3::HTTPAccessLogEntry&) { - // TODO(shikugawa): To buffer message it determined as critical message, - // by the trigger of configured thresholds. e.g. matched specific filter - // which describes https log message has >= 500 status code. - return true; -} - -bool GrpcAccessLoggerImpl::shouldBuffer(const envoy::data::accesslog::v3::TCPAccessLogEntry&) { - // TODO(shikugawa): Not supported for TCP message. - return false; -} - bool GrpcAccessLoggerImpl::isEmpty() { return !message_.has_http_logs() && !message_.has_tcp_logs(); } +bool GrpcAccessLoggerImpl::isFatalEmpty() { + return !fatal_message_.has_http_logs() && !fatal_message_.has_tcp_logs(); +} + void GrpcAccessLoggerImpl::initMessage() { auto* identifier = message_.mutable_identifier(); *identifier->mutable_node() = local_info_.node(); identifier->set_log_name(log_name_); } +void GrpcAccessLoggerImpl::initFatalMessage() { + auto* identifier = fatal_message_.mutable_identifier(); + *identifier->mutable_node() = local_info_.node(); + identifier->set_log_name(log_name_); +} + GrpcAccessLoggerCacheImpl::GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& async_client_manager, Stats::Scope& scope, ThreadLocal::SlotAllocator& tls, diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 9587d925e31b9..0c66d6e0a37b2 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -35,10 +35,10 @@ class GrpcAccessLoggerImpl void addEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) override; void addFatalEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) override; void addFatalEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) override; - bool shouldBuffer(const envoy::data::accesslog::v3::HTTPAccessLogEntry& entry) override; - bool shouldBuffer(const envoy::data::accesslog::v3::TCPAccessLogEntry& entry) override; bool isEmpty() override; + bool isFatalEmpty() override; void initMessage() override; + void initFatalMessage() override; const std::string log_name_; const LocalInfo::LocalInfo& local_info_; diff --git a/source/extensions/access_loggers/grpc/http_config.cc b/source/extensions/access_loggers/grpc/http_config.cc index 4d333f6d91fdc..100d7f27ad68e 100644 --- a/source/extensions/access_loggers/grpc/http_config.cc +++ b/source/extensions/access_loggers/grpc/http_config.cc @@ -29,7 +29,7 @@ AccessLog::InstanceSharedPtr HttpGrpcAccessLogFactory::createAccessLogInstance( return std::make_shared(std::move(filter), proto_config, context.threadLocal(), GrpcCommon::getGrpcAccessLoggerCacheSingleton(context), - context.scope()); + context.scope(), context); } ProtobufTypes::MessagePtr HttpGrpcAccessLogFactory::createEmptyConfigProto() { diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc index cb37f1315efd2..979ef61b059d9 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc @@ -1,5 +1,7 @@ #include "source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h" +#include + #include "envoy/config/core/v3/base.pb.h" #include "envoy/data/accesslog/v3/accesslog.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" @@ -27,7 +29,7 @@ HttpGrpcAccessLog::HttpGrpcAccessLog( AccessLog::FilterPtr&& filter, envoy::extensions::access_loggers::grpc::v3::HttpGrpcAccessLogConfig config, ThreadLocal::SlotAllocator& tls, GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache, - Stats::Scope& scope) + Stats::Scope& scope, Server::Configuration::CommonFactoryContext& context) : Common::ImplBase(std::move(filter)), scope_(scope), config_(std::move(config)), tls_slot_(tls.allocateSlot()), access_logger_cache_(std::move(access_logger_cache)) { for (const auto& header : config_.additional_request_headers_to_log()) { @@ -47,6 +49,12 @@ HttpGrpcAccessLog::HttpGrpcAccessLog( return std::make_shared(access_logger_cache_->getOrCreateLogger( config_.common_config(), transport_version, Common::GrpcAccessLoggerType::HTTP, scope_)); }); + + if (config.has_common_config() && config.common_config().has_buffer_log_filter()) { + critical_log_filter_ = AccessLog::FilterFactory::fromProto( + config.common_config().buffer_log_filter(), context.runtime(), + context.api().randomGenerator(), context.messageValidationVisitor()); + } } void HttpGrpcAccessLog::emitLog(const Http::RequestHeaderMap& request_headers, @@ -161,7 +169,13 @@ void HttpGrpcAccessLog::emitLog(const Http::RequestHeaderMap& request_headers, } } - tls_slot_->getTyped().logger_->log(std::move(log_entry)); + bool is_critical = false; + if (critical_log_filter_) { + is_critical = critical_log_filter_->evaluate(stream_info, request_headers, response_headers, + response_trailers); + } + + tls_slot_->getTyped().logger_->log(std::move(log_entry), is_critical); } } // namespace HttpGrpc diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h index d403596f8decc..d56a7c248efe2 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h @@ -30,7 +30,7 @@ class HttpGrpcAccessLog : public Common::ImplBase { envoy::extensions::access_loggers::grpc::v3::HttpGrpcAccessLogConfig config, ThreadLocal::SlotAllocator& tls, GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache, - Stats::Scope& scope); + Stats::Scope& scope, Server::Configuration::CommonFactoryContext& context); private: /** @@ -56,6 +56,7 @@ class HttpGrpcAccessLog : public Common::ImplBase { std::vector response_headers_to_log_; std::vector response_trailers_to_log_; std::vector filter_states_to_log_; + AccessLog::FilterPtr critical_log_filter_; }; using HttpGrpcAccessLogPtr = std::unique_ptr; diff --git a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc index c475f134a7b61..66bcd999653ff 100644 --- a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc @@ -45,7 +45,8 @@ void TcpGrpcAccessLog::emitLog(const Http::RequestHeaderMap&, const Http::Respon connection_properties.set_sent_bytes(stream_info.bytesSent()); // request_properties->set_request_body_bytes(stream_info.bytesReceived()); - tls_slot_->getTyped().logger_->log(std::move(log_entry)); + // TODO(shikugawa): implement TCP gRPC access critical message buffering. + tls_slot_->getTyped().logger_->log(std::move(log_entry), false); } } // namespace TcpGrpc diff --git a/source/extensions/access_loggers/open_telemetry/access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/access_log_impl.cc index f3376483ad318..8a061de81db4c 100644 --- a/source/extensions/access_loggers/open_telemetry/access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/access_log_impl.cc @@ -75,7 +75,7 @@ void AccessLog::emitLog(const Http::RequestHeaderMap& request_headers, attributes); *log_entry.mutable_attributes() = attributes.values(); - tls_slot_->getTyped().logger_->log(std::move(log_entry)); + tls_slot_->getTyped().logger_->log(std::move(log_entry), false); } } // namespace OpenTelemetry diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc index cc84242c93d67..2f8efd089f806 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc @@ -65,6 +65,11 @@ void GrpcAccessLoggerImpl::addEntry(opentelemetry::proto::logs::v1::LogRecord&& bool GrpcAccessLoggerImpl::isEmpty() { return root_->logs().empty(); } +bool GrpcAccessLoggerImpl::isFatalEmpty() { + // Critical message must be empty in OpenTelemetry logger. + return true; +} + // The message is already initialized in the c'tor, and only the logs are cleared. void GrpcAccessLoggerImpl::initMessage() {} diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h index 2b7024d102046..d254154214f9f 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h @@ -49,11 +49,11 @@ class GrpcAccessLoggerImpl void addEntry(ProtobufWkt::Empty&& entry) override { (void)entry; }; void addFatalEntry(opentelemetry::proto::logs::v1::LogRecord&&) override {} void addFatalEntry(ProtobufWkt::Empty&&) override{}; - bool shouldBuffer(const opentelemetry::proto::logs::v1::LogRecord&) override { return false; } - bool shouldBuffer(const ProtobufWkt::Empty&) override { return false; } bool isEmpty() override; + bool isFatalEmpty() override; void initMessage() override; void clearMessage() override; + void initFatalMessage() override {} opentelemetry::proto::logs::v1::InstrumentationLibraryLogs* root_; }; diff --git a/test/extensions/access_loggers/common/grpc_access_logger_test.cc b/test/extensions/access_loggers/common/grpc_access_logger_test.cc index bf47a0e6c6cab..ee75e710e3377 100644 --- a/test/extensions/access_loggers/common/grpc_access_logger_test.cc +++ b/test/extensions/access_loggers/common/grpc_access_logger_test.cc @@ -109,11 +109,10 @@ class MockGrpcAccessLoggerImpl (void)entry; mockAddFatalEntry(MOCK_TCP_LOG_FIELD_NAME); } - bool shouldBuffer(const ProtobufWkt::Struct&) override { return http_should_buffer_; } - bool shouldBuffer(const ProtobufWkt::Empty&) override { return tcp_should_buffer_; } bool isEmpty() override { return message_.fields().empty(); } - + bool isFatalEmpty() override { return fatal_message_.fields().empty(); } + void initFatalMessage() override { ++num_fatal_inits_; } void initMessage() override { ++num_inits_; } void clearMessage() override { @@ -121,9 +120,8 @@ class MockGrpcAccessLoggerImpl num_clears_++; } - bool tcp_should_buffer_ = false; - bool http_should_buffer_ = false; int num_inits_ = 0; + int num_fatal_inits_ = 0; int num_clears_ = 0; }; @@ -189,7 +187,7 @@ TEST_F(GrpcAccessLogTest, BasicFlow) { expectStreamStart(stream, &callbacks); // Log an HTTP entry. expectFlushedLogEntriesCount(stream, MOCK_HTTP_LOG_FIELD_NAME, 1); - logger_->log(mockHttpEntry()); + logger_->log(mockHttpEntry(), false); EXPECT_EQ(1, logger_->numInits()); // Messages should be cleared after each flush. EXPECT_EQ(1, logger_->numClears()); @@ -198,7 +196,7 @@ TEST_F(GrpcAccessLogTest, BasicFlow) { // Log a TCP entry. expectFlushedLogEntriesCount(stream, MOCK_TCP_LOG_FIELD_NAME, 1); - logger_->log(ProtobufWkt::Empty()); + logger_->log(ProtobufWkt::Empty(), false); EXPECT_EQ(2, logger_->numClears()); // TCP logging doesn't change the logs_written counter. EXPECT_EQ(1, @@ -213,7 +211,7 @@ TEST_F(GrpcAccessLogTest, BasicFlow) { expectStreamStart(stream, &callbacks); // Log an HTTP entry. expectFlushedLogEntriesCount(stream, MOCK_HTTP_LOG_FIELD_NAME, 1); - logger_->log(mockHttpEntry()); + logger_->log(mockHttpEntry(), false); // Message should be initialized again. EXPECT_EQ(2, logger_->numInits()); EXPECT_EQ(3, logger_->numClears()); @@ -235,7 +233,7 @@ TEST_F(GrpcAccessLogTest, WatermarksOverrun) { // Fail to flush, so the log stays buffered up. EXPECT_CALL(stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(true)); EXPECT_CALL(stream, sendMessageRaw_(_, false)).Times(0); - logger_->log(mockHttpEntry()); + logger_->log(mockHttpEntry(), false); // No entry was logged so no clear expected. EXPECT_EQ(0, logger_->numClears()); EXPECT_EQ(1, @@ -246,7 +244,7 @@ TEST_F(GrpcAccessLogTest, WatermarksOverrun) { // Now canLogMore will fail, and the next log will be dropped. EXPECT_CALL(stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(true)); EXPECT_CALL(stream, sendMessageRaw_(_, _)).Times(0); - logger_->log(mockHttpEntry()); + logger_->log(mockHttpEntry(), false); EXPECT_EQ(1, logger_->numInits()); // Still no entry was logged so no clear expected. EXPECT_EQ(0, logger_->numClears()); @@ -261,7 +259,7 @@ TEST_F(GrpcAccessLogTest, WatermarksOverrun) { EXPECT_CALL(stream, sendMessageRaw_(_, _)); EXPECT_CALL(stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); EXPECT_CALL(stream, sendMessageRaw_(_, _)); - logger_->log(mockHttpEntry()); + logger_->log(mockHttpEntry(), false); // Now both entries were logged separately so we expect 2 clears. EXPECT_EQ(2, logger_->numClears()); EXPECT_EQ(2, @@ -281,7 +279,7 @@ TEST_F(GrpcAccessLogTest, StreamFailure) { callbacks.onRemoteClose(Grpc::Status::Internal, "bad"); return nullptr; })); - logger_->log(mockHttpEntry()); + logger_->log(mockHttpEntry(), false); EXPECT_EQ(1, logger_->numInits()); } @@ -296,9 +294,9 @@ TEST_F(GrpcAccessLogTest, Batching) { expectStreamStart(stream, &callbacks); expectFlushedLogEntriesCount(stream, MOCK_HTTP_LOG_FIELD_NAME, 3); - logger_->log(mockHttpEntry()); - logger_->log(mockHttpEntry()); - logger_->log(mockHttpEntry()); + logger_->log(mockHttpEntry(), false); + logger_->log(mockHttpEntry(), false); + logger_->log(mockHttpEntry(), false); EXPECT_EQ(1, logger_->numInits()); // The entries were batched and logged together so we expect a single clear. EXPECT_EQ(1, logger_->numClears()); @@ -308,7 +306,7 @@ TEST_F(GrpcAccessLogTest, Batching) { ProtobufWkt::Struct big_entry = mockHttpEntry(); const std::string big_key(max_buffer_size, 'a'); big_entry.mutable_fields()->insert({big_key, ProtobufWkt::Value()}); - logger_->log(std::move(big_entry)); + logger_->log(std::move(big_entry), false); EXPECT_EQ(2, logger_->numClears()); } @@ -321,7 +319,7 @@ TEST_F(GrpcAccessLogTest, Flushing) { timer_->invokeCallback(); // Not enough data yet to trigger flush on batch size. - logger_->log(mockHttpEntry()); + logger_->log(mockHttpEntry(), false); MockAccessLogStream stream; AccessLogCallbacks* callbacks; diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index eec260b9c347b..4a97003153034 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -100,7 +100,7 @@ TEST_F(GrpcAccessLoggerImplTest, LogHttp) { )EOF"); envoy::data::accesslog::v3::HTTPAccessLogEntry entry; entry.mutable_request()->set_path("/test/path1"); - logger_->log(envoy::data::accesslog::v3::HTTPAccessLogEntry(entry)); + logger_->log(envoy::data::accesslog::v3::HTTPAccessLogEntry(entry), false); } TEST_F(GrpcAccessLoggerImplTest, LogTcp) { @@ -119,7 +119,7 @@ TEST_F(GrpcAccessLoggerImplTest, LogTcp) { )EOF"); envoy::data::accesslog::v3::TCPAccessLogEntry tcp_entry; tcp_entry.mutable_common_properties()->set_sample_rate(1); - logger_->log(envoy::data::accesslog::v3::TCPAccessLogEntry(tcp_entry)); + logger_->log(envoy::data::accesslog::v3::TCPAccessLogEntry(tcp_entry), false); } class GrpcAccessLoggerCacheImplTest : public testing::Test { @@ -173,7 +173,7 @@ TEST_F(GrpcAccessLoggerCacheImplTest, LoggerCreation) { )EOF"); envoy::data::accesslog::v3::HTTPAccessLogEntry entry; entry.mutable_request()->set_path("/test/path1"); - logger->log(envoy::data::accesslog::v3::HTTPAccessLogEntry(entry)); + logger->log(envoy::data::accesslog::v3::HTTPAccessLogEntry(entry), false); } } // namespace diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index 54e012875513c..5e604f066377b 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -12,6 +12,7 @@ #include "test/mocks/access_log/mocks.h" #include "test/mocks/grpc/mocks.h" #include "test/mocks/local_info/mocks.h" +#include "test/mocks/server/factory_context.h" #include "test/mocks/ssl/mocks.h" #include "test/mocks/stream_info/mocks.h" #include "test/mocks/thread_local/mocks.h" @@ -36,8 +37,9 @@ using envoy::data::accesslog::v3::HTTPAccessLogEntry; class MockGrpcAccessLogger : public GrpcCommon::GrpcAccessLogger { public: // GrpcAccessLogger - MOCK_METHOD(void, log, (HTTPAccessLogEntry && entry)); - MOCK_METHOD(void, log, (envoy::data::accesslog::v3::TCPAccessLogEntry && entry)); + MOCK_METHOD(void, log, (HTTPAccessLogEntry && entry, bool is_critical)); + MOCK_METHOD(void, log, + (envoy::data::accesslog::v3::TCPAccessLogEntry && entry, bool is_critical)); }; class MockGrpcAccessLoggerCache : public GrpcCommon::GrpcAccessLoggerCache { @@ -70,7 +72,7 @@ class HttpGrpcAccessLogTest : public testing::Test { return logger_; }); access_log_ = std::make_unique(AccessLog::FilterPtr{filter_}, config_, tls_, - logger_cache_, scope_); + logger_cache_, scope_, factory_context_); } void expectLog(const std::string& expected_log_entry_yaml) { @@ -80,9 +82,9 @@ class HttpGrpcAccessLogTest : public testing::Test { HTTPAccessLogEntry expected_log_entry; TestUtility::loadFromYaml(expected_log_entry_yaml, expected_log_entry); - EXPECT_CALL(*logger_, log(An())) - .WillOnce( - Invoke([expected_log_entry](envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { + EXPECT_CALL(*logger_, log(An(), _)) + .WillOnce(Invoke( + [expected_log_entry](envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry, bool) { EXPECT_EQ(entry.DebugString(), expected_log_entry.DebugString()); })); } @@ -128,6 +130,7 @@ response: {{}} std::shared_ptr logger_{new MockGrpcAccessLogger()}; std::shared_ptr logger_cache_{new MockGrpcAccessLoggerCache()}; HttpGrpcAccessLogPtr access_log_; + NiceMock factory_context_; }; class TestSerializedFilterState : public StreamInfo::FilterState::Object { diff --git a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc index d169f040f3b9f..9d585a9be4333 100644 --- a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc @@ -29,6 +29,7 @@ using ::Envoy::AccessLog::MockFilter; using opentelemetry::proto::logs::v1::LogRecord; using testing::_; using testing::An; +using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::NiceMock; @@ -43,8 +44,8 @@ namespace { class MockGrpcAccessLogger : public GrpcAccessLogger { public: // GrpcAccessLogger - MOCK_METHOD(void, log, (LogRecord && entry)); - MOCK_METHOD(void, log, (ProtobufWkt::Empty && entry)); + MOCK_METHOD(void, log, (LogRecord && entry, bool)); + MOCK_METHOD(void, log, (ProtobufWkt::Empty && entry, bool)); }; class MockGrpcAccessLoggerCache : public GrpcAccessLoggerCache { @@ -104,8 +105,8 @@ string_value: "x-request-header: %REQ(x-request-header)%, protocol: %PROTOCOL%" LogRecord expected_log_entry; TestUtility::loadFromYaml(expected_log_entry_yaml, expected_log_entry); - EXPECT_CALL(*logger_, log(An())) - .WillOnce(Invoke([expected_log_entry](LogRecord&& entry) { + EXPECT_CALL(*logger_, log(An(), Eq(false))) + .WillOnce(Invoke([expected_log_entry](LogRecord&& entry, bool) { EXPECT_EQ(entry.DebugString(), expected_log_entry.DebugString()); })); } diff --git a/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc index ecd8d6b96a83e..743d634e69db5 100644 --- a/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc @@ -116,7 +116,7 @@ TEST_F(GrpcAccessLoggerImplTest, LogHttp) { )EOF"); opentelemetry::proto::logs::v1::LogRecord entry; entry.set_severity_text("test-severity-text"); - logger_->log(opentelemetry::proto::logs::v1::LogRecord(entry)); + logger_->log(opentelemetry::proto::logs::v1::LogRecord(entry), false); } TEST_F(GrpcAccessLoggerImplTest, LogTcp) { @@ -142,7 +142,7 @@ TEST_F(GrpcAccessLoggerImplTest, LogTcp) { )EOF"); opentelemetry::proto::logs::v1::LogRecord entry; entry.set_severity_text("test-severity-text"); - logger_->log(opentelemetry::proto::logs::v1::LogRecord(entry)); + logger_->log(opentelemetry::proto::logs::v1::LogRecord(entry), false); } class GrpcAccessLoggerCacheImplTest : public testing::Test { @@ -202,7 +202,7 @@ TEST_F(GrpcAccessLoggerCacheImplTest, LoggerCreation) { )EOF"); opentelemetry::proto::logs::v1::LogRecord entry; entry.set_severity_text("test-severity-text"); - logger->log(opentelemetry::proto::logs::v1::LogRecord(entry)); + logger->log(opentelemetry::proto::logs::v1::LogRecord(entry), false); } } // namespace From a63ec679a4aa0d7a1cabcf2a520c6025f11434f1 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Thu, 29 Jul 2021 09:39:22 +0000 Subject: [PATCH 03/39] wip Signed-off-by: Shikugawa --- api/envoy/service/accesslog/v3/als.proto | 4 + api/envoy/service/accesslog/v4alpha/als.proto | 4 + .../envoy/service/accesslog/v3/als.proto | 4 + .../envoy/service/accesslog/v4alpha/als.proto | 4 + .../common/grpc_access_logger.h | 134 +++--------------- .../grpc/grpc_access_log_impl.cc | 2 +- .../grpc/grpc_access_log_impl.h | 111 +++++++++++++++ .../open_telemetry/grpc_access_log_impl.h | 1 + .../common/grpc_access_logger_test.cc | 6 + 9 files changed, 156 insertions(+), 114 deletions(-) diff --git a/api/envoy/service/accesslog/v3/als.proto b/api/envoy/service/accesslog/v3/als.proto index f6d8a5531ad81..ff25f185aedf7 100644 --- a/api/envoy/service/accesslog/v3/als.proto +++ b/api/envoy/service/accesslog/v3/als.proto @@ -105,4 +105,8 @@ message StreamAccessLogsMessage { TCPAccessLogEntries tcp_logs = 3; } + + // This is an ID to identify the message, and can be added to the Critical Endpoint + // response message to uniquely identify the message being buffered in the Envoy. + uint32 id = 4; } diff --git a/api/envoy/service/accesslog/v4alpha/als.proto b/api/envoy/service/accesslog/v4alpha/als.proto index cf91576a86306..e50f9a96b1e96 100644 --- a/api/envoy/service/accesslog/v4alpha/als.proto +++ b/api/envoy/service/accesslog/v4alpha/als.proto @@ -108,4 +108,8 @@ message StreamAccessLogsMessage { TCPAccessLogEntries tcp_logs = 3; } + + // This is an ID to identify the message, and can be added to the Critical Endpoint + // response message to uniquely identify the message being buffered in the Envoy. + uint32 id = 4; } diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto index f6d8a5531ad81..ff25f185aedf7 100644 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v3/als.proto @@ -105,4 +105,8 @@ message StreamAccessLogsMessage { TCPAccessLogEntries tcp_logs = 3; } + + // This is an ID to identify the message, and can be added to the Critical Endpoint + // response message to uniquely identify the message being buffered in the Envoy. + uint32 id = 4; } diff --git a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto index cf91576a86306..e50f9a96b1e96 100644 --- a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto @@ -108,4 +108,8 @@ message StreamAccessLogsMessage { TCPAccessLogEntries tcp_logs = 3; } + + // This is an ID to identify the message, and can be added to the Critical Endpoint + // response message to uniquely identify the message being buffered in the Envoy. + uint32 id = 4; } diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index ef062996ccd1c..db66652511592 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -150,108 +150,6 @@ template class GrpcAccessLogClient { const absl::optional transport_api_version_; }; -template class FatalAccessLoggerGrpcClient { -public: - using ResponseType = envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse; - - FatalAccessLoggerGrpcClient(const Grpc::RawAsyncClientSharedPtr& client, - const Protobuf::MethodDescriptor& method) - : FatalAccessLoggerGrpcClient(client, method, absl::nullopt) {} - FatalAccessLoggerGrpcClient( - const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& method, - absl::optional transport_api_version) - : client_(client), service_method_(method), transport_api_version_(transport_api_version) {} - - struct BufferedMessage { - enum class State { - Initial, - Pending, - }; - - State state_{State::Initial}; - ResponseType& message_; - }; - - struct ActiveStream : public Grpc::AsyncStreamCallbacks { - ActiveStream(FatalAccessLoggerGrpcClient& parent) : parent_(parent) {} - - // Grpc::AsyncStreamCallbacks - void onCreateInitialMetadata(Http::RequestHeaderMap&) override {} - void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&&) override {} - void onReceiveMessage(std::unique_ptr&& message) override { - const auto& id = message->id(); - - if (parent_.buffered_messages_.find(id) != parent_.buffered_messages_.end()) { - switch (message->status()) { - case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK: - parent_.buffered_messages_.erase(id); - break; - case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK: - parent_.buffered_messages_.at(id).state_ = BufferedMessage::State::Initial; - // TODO(shikugawa): After many NACK occurred, Buffer will be overwhelmed. - // To resolve this, Buffer should have size limit. - break; - default: - return; - } - } - } - void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {} - void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override { - ASSERT(parent_.active_stream_ != nullptr); - if (parent_.active_stream_->stream_ != nullptr) { - parent_.active_stream_.reset(); - } - } - - FatalAccessLoggerGrpcClient& parent_; - Grpc::AsyncStream stream_; - }; - - void flush() { - if (!active_stream_) { - active_stream_ = std::make_unique(*this); - } - - if (active_stream_->stream_ == nullptr) { - active_stream_->stream_ = - client_.start(service_method_, *active_stream_, Http::AsyncClient::StreamOptions()); - } - - if (active_stream_->stream_ == nullptr || - active_stream_->stream_.isAboveWriteBufferHighWatermark()) { - active_stream_.reset(); - return; - } - - for (auto&& buffered_message : buffered_messages_) { - uint32_t id = buffered_message.first; - buffered_message.second.message_.set_id(id); - buffered_message.second.state_ = BufferedMessage::State::Pending; - - // TODO(shikugawa): stream must be ended with response withing specified timeout, - // that must be defined to setup duration after last message transported. - if (transport_api_version_.has_value()) { - active_stream_->stream_->sendMessage(buffered_message.second.message_, - transport_api_version_.value(), false); - } else { - active_stream_->stream_->sendMessage(buffered_message.second.message_, false); - } - } - } - - bool isStreamStarted() { return active_stream_ != nullptr && active_stream_->stream_ != nullptr; } - -private: - friend ActiveStream; - - absl::flat_hash_map buffered_messages_; - std::unique_ptr active_stream_; - Grpc::AsyncClient client_; - const Protobuf::MethodDescriptor& service_method_; - const absl::optional transport_api_version_; -}; - } // namespace Detail /** @@ -260,6 +158,7 @@ template class FatalAccessLoggerGrpcClient { #define ALL_GRPC_ACCESS_LOGGER_STATS(COUNTER) \ COUNTER(logs_written) \ COUNTER(logs_dropped) +// TODO(shikugawa): implement critical message related stats. /** * Wrapper struct for the access log stats. @see stats_macros.h @@ -268,6 +167,21 @@ struct GrpcAccessLoggerStats { ALL_GRPC_ACCESS_LOGGER_STATS(GENERATE_COUNTER_STRUCT) }; +template class CriticalAccessLoggerGrpcClient { +public: + virtual ~CriticalAccessLoggerGrpcClient() = default; + + /** + * Flush critical messages. + */ + virtual void flush(RequestType message) PURE; + + /** + * Whether this client has active stream or not. + */ + virtual bool isStreamStarted() PURE; +}; + /** * Base class for defining a gRPC logger with the `HttpLogProto` and `TcpLogProto` access log * entries and `LogRequest` and `LogResponse` gRPC messages. @@ -340,15 +254,8 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger>( - client, service_method, transport_api_version); - } - Detail::GrpcAccessLogClient client_; - std::unique_ptr> fatal_client_; + std::unique_ptr> fatal_client_; LogRequest message_; LogRequest fatal_message_; @@ -362,6 +269,7 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerisStreamStarted()) { initFatalMessage(); } - fatal_client_->flush(); - // TODO(shikugawa): consider message and buffer lifecycle. + fatal_client_->flush(fatal_message_); + clearFatalMessage(); } bool canLogMore() { diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 0f78ba3be7eea..92ccfecb1234b 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -28,7 +28,7 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( .getMethodDescriptorForVersion(transport_api_version), transport_api_version), log_name_(log_name), local_info_(local_info) { - initFatalLoggerClient( + fatal_client_ = std::make_unique( client, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs"), diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 0c66d6e0a37b2..7b5946ba778de 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -17,6 +17,117 @@ namespace Extensions { namespace AccessLoggers { namespace GrpcCommon { +class CriticalAccessLoggerGrpcClientImpl + : public Common::CriticalAccessLoggerGrpcClient< + envoy::service::accesslog::v3::StreamAccessLogsMessage> { +public: + using RequestType = envoy::service::accesslog::v3::StreamAccessLogsMessage; + using ResponseType = envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse; + + CriticalAccessLoggerGrpcClientImpl(const Grpc::RawAsyncClientSharedPtr& client, + const Protobuf::MethodDescriptor& method) + : CriticalAccessLoggerGrpcClientImpl(client, method, absl::nullopt) {} + CriticalAccessLoggerGrpcClientImpl( + const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& method, + absl::optional transport_api_version) + : client_(client), service_method_(method), transport_api_version_(transport_api_version) {} + + struct BufferedMessage { + enum class State { + Initial, + Pending, + }; + + State state_{State::Initial}; + RequestType message_; + }; + + struct ActiveStream : public Grpc::AsyncStreamCallbacks { + ActiveStream(CriticalAccessLoggerGrpcClientImpl& parent) : parent_(parent) {} + + // Grpc::AsyncStreamCallbacks + void onCreateInitialMetadata(Http::RequestHeaderMap&) override {} + void onReceiveInitialMetadata(Http::ResponseHeaderMapPtr&&) override {} + void onReceiveMessage(std::unique_ptr&& message) override { + const auto& id = message->id(); + + if (parent_.buffered_messages_.find(id) != parent_.buffered_messages_.end()) { + switch (message->status()) { + case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK: + parent_.buffered_messages_.erase(id); + break; + case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK: + parent_.buffered_messages_.at(id).state_ = BufferedMessage::State::Initial; + // TODO(shikugawa): After many NACK occurred, Buffer will be overwhelmed. + // To resolve this, Buffer should have size limit. + break; + default: + return; + } + } + } + void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {} + void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override { + ASSERT(parent_.active_stream_ != nullptr); + if (parent_.active_stream_->stream_ != nullptr) { + parent_.active_stream_.reset(); + } + } + + CriticalAccessLoggerGrpcClientImpl& parent_; + Grpc::AsyncStream stream_; + }; + + // Copy messages in the buffer. Take care about memory pressure. + void flush(RequestType message) override { + if (!active_stream_) { + active_stream_ = std::make_unique(*this); + } + + if (active_stream_->stream_ == nullptr) { + active_stream_->stream_ = + client_.start(service_method_, *active_stream_, Http::AsyncClient::StreamOptions()); + } + + if (active_stream_->stream_ == nullptr || + active_stream_->stream_.isAboveWriteBufferHighWatermark()) { + active_stream_.reset(); + return; + } + + uint32_t id = MessageUtil::hash(message); + buffered_messages_[id] = BufferedMessage{BufferedMessage::State::Initial, message}; + + for (auto&& buffered_message : buffered_messages_) { + uint32_t id = buffered_message.first; + buffered_message.second.message_.set_id(id); + buffered_message.second.state_ = BufferedMessage::State::Pending; + + // TODO(shikugawa): stream must be ended with response withing specified timeout, + // that must be defined to setup duration after last message transported. + if (transport_api_version_.has_value()) { + active_stream_->stream_->sendMessage(buffered_message.second.message_, + transport_api_version_.value(), false); + } else { + active_stream_->stream_->sendMessage(buffered_message.second.message_, false); + } + } + } + + bool isStreamStarted() override { + return active_stream_ != nullptr && active_stream_->stream_ != nullptr; + } + +private: + friend ActiveStream; + + absl::flat_hash_map buffered_messages_; + std::unique_ptr active_stream_; + Grpc::AsyncClient client_; + const Protobuf::MethodDescriptor& service_method_; + const absl::optional transport_api_version_; +}; + class GrpcAccessLoggerImpl : public Common::GrpcAccessLogger Date: Thu, 29 Jul 2021 10:58:31 +0000 Subject: [PATCH 04/39] timer Signed-off-by: Shikugawa --- .../common/grpc_access_logger.h | 7 +-- .../grpc/grpc_access_log_impl.cc | 14 +++--- .../grpc/grpc_access_log_impl.h | 43 ++++++++++++++----- .../access_loggers/grpc/http_config.cc | 4 +- .../grpc/http_grpc_access_log_impl.cc | 13 +++--- .../grpc/http_grpc_access_log_impl.h | 3 +- .../grpc/grpc_access_log_impl_test.cc | 4 +- .../grpc/http_grpc_access_log_impl_test.cc | 4 +- 8 files changed, 57 insertions(+), 35 deletions(-) diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index db66652511592..b97aa249eed2d 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -1,16 +1,11 @@ #pragma once -#include - #include -#include -// TODO(shikugagwa): extensions should not be placed here -#include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" -#include "envoy/service/accesslog/v3/als.pb.h" #include "envoy/config/core/v3/config_source.pb.h" #include "envoy/event/dispatcher.h" #include "envoy/grpc/async_client_manager.h" +#include "envoy/service/accesslog/v3/als.pb.h" #include "envoy/singleton/instance.h" #include "envoy/stats/scope.h" #include "envoy/thread_local/thread_local.h" diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 92ccfecb1234b..4168f0bebec9f 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -1,5 +1,7 @@ #include "source/extensions/access_loggers/grpc/grpc_access_log_impl.h" +#include + #include "envoy/data/accesslog/v3/accesslog.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" #include "envoy/grpc/async_client_manager.h" @@ -16,7 +18,8 @@ namespace AccessLoggers { namespace GrpcCommon { GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( - const Grpc::RawAsyncClientSharedPtr& client, std::string log_name, + const Grpc::RawAsyncClientSharedPtr& client, + const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& config, std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, envoy::config::core::v3::ApiVersion transport_api_version) @@ -27,11 +30,12 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( "envoy.service.accesslog.v2.AccessLogService.StreamAccessLogs") .getMethodDescriptorForVersion(transport_api_version), transport_api_version), - log_name_(log_name), local_info_(local_info) { + log_name_(config.log_name()), local_info_(local_info) { fatal_client_ = std::make_unique( client, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs"), + dispatcher, PROTOBUF_GET_MS_OR_DEFAULT(config, message_ack_timeout, 5000), transport_api_version); } @@ -83,9 +87,9 @@ GrpcAccessLoggerImpl::SharedPtr GrpcAccessLoggerCacheImpl::createLogger( const Grpc::RawAsyncClientSharedPtr& client, std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, Stats::Scope& scope) { - return std::make_shared(client, config.log_name(), - buffer_flush_interval_msec, max_buffer_size_bytes, - dispatcher, local_info_, scope, transport_version); + return std::make_shared(client, config, buffer_flush_interval_msec, + max_buffer_size_bytes, dispatcher, local_info_, + scope, transport_version); } } // namespace GrpcCommon diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 7b5946ba778de..9ad0922390d3f 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "envoy/data/accesslog/v3/accesslog.pb.h" @@ -25,12 +26,16 @@ class CriticalAccessLoggerGrpcClientImpl using ResponseType = envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse; CriticalAccessLoggerGrpcClientImpl(const Grpc::RawAsyncClientSharedPtr& client, - const Protobuf::MethodDescriptor& method) - : CriticalAccessLoggerGrpcClientImpl(client, method, absl::nullopt) {} + const Protobuf::MethodDescriptor& method, + Event::Dispatcher& dispatcher, uint64_t message_ack_timeout) + : CriticalAccessLoggerGrpcClientImpl(client, method, dispatcher, message_ack_timeout, + absl::nullopt) {} CriticalAccessLoggerGrpcClientImpl( const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& method, + Event::Dispatcher& dispatcher, uint64_t message_ack_timeout, absl::optional transport_api_version) - : client_(client), service_method_(method), transport_api_version_(transport_api_version) {} + : client_(client), service_method_(method), dispatcher_(dispatcher), + message_ack_timeout_(message_ack_timeout), transport_api_version_(transport_api_version) {} struct BufferedMessage { enum class State { @@ -38,6 +43,7 @@ class CriticalAccessLoggerGrpcClientImpl Pending, }; + Event::TimerPtr timer_; State state_{State::Initial}; RequestType message_; }; @@ -52,6 +58,11 @@ class CriticalAccessLoggerGrpcClientImpl const auto& id = message->id(); if (parent_.buffered_messages_.find(id) != parent_.buffered_messages_.end()) { + // After response wait time exceeded, the state should be Initial. + if (parent_.buffered_messages_.at(id).state_ != BufferedMessage::State::Pending) { + return; + } + switch (message->status()) { case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK: parent_.buffered_messages_.erase(id); @@ -96,15 +107,22 @@ class CriticalAccessLoggerGrpcClientImpl } uint32_t id = MessageUtil::hash(message); - buffered_messages_[id] = BufferedMessage{BufferedMessage::State::Initial, message}; + buffered_messages_[id] = BufferedMessage{nullptr, BufferedMessage::State::Initial, message}; for (auto&& buffered_message : buffered_messages_) { uint32_t id = buffered_message.first; buffered_message.second.message_.set_id(id); buffered_message.second.state_ = BufferedMessage::State::Pending; - // TODO(shikugawa): stream must be ended with response withing specified timeout, - // that must be defined to setup duration after last message transported. + buffered_message.second.timer_ = dispatcher_.createTimer([&buffered_message]() { + if (buffered_message.second.state_ == BufferedMessage::State::Pending) { + buffered_message.second.state_ = BufferedMessage::State::Initial; + buffered_message.second.timer_->disableTimer(); + buffered_message.second.timer_.reset(); + } + }); + buffered_message.second.timer_->enableTimer(message_ack_timeout_); + if (transport_api_version_.has_value()) { active_stream_->stream_->sendMessage(buffered_message.second.message_, transport_api_version_.value(), false); @@ -125,6 +143,8 @@ class CriticalAccessLoggerGrpcClientImpl std::unique_ptr active_stream_; Grpc::AsyncClient client_; const Protobuf::MethodDescriptor& service_method_; + Event::Dispatcher& dispatcher_; + std::chrono::milliseconds message_ack_timeout_; const absl::optional transport_api_version_; }; @@ -134,11 +154,12 @@ class GrpcAccessLoggerImpl envoy::service::accesslog::v3::StreamAccessLogsMessage, envoy::service::accesslog::v3::StreamAccessLogsResponse> { public: - GrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, std::string log_name, - std::chrono::milliseconds buffer_flush_interval_msec, - uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, - const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, - envoy::config::core::v3::ApiVersion transport_api_version); + GrpcAccessLoggerImpl( + const Grpc::RawAsyncClientSharedPtr& client, + const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& config, + std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, + Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, + envoy::config::core::v3::ApiVersion transport_api_version); private: // Extensions::AccessLoggers::GrpcCommon::GrpcAccessLogger diff --git a/source/extensions/access_loggers/grpc/http_config.cc b/source/extensions/access_loggers/grpc/http_config.cc index 100d7f27ad68e..fd1cf05027e68 100644 --- a/source/extensions/access_loggers/grpc/http_config.cc +++ b/source/extensions/access_loggers/grpc/http_config.cc @@ -27,9 +27,9 @@ AccessLog::InstanceSharedPtr HttpGrpcAccessLogFactory::createAccessLogInstance( const envoy::extensions::access_loggers::grpc::v3::HttpGrpcAccessLogConfig&>( config, context.messageValidationVisitor()); - return std::make_shared(std::move(filter), proto_config, context.threadLocal(), + return std::make_shared(std::move(filter), proto_config, GrpcCommon::getGrpcAccessLoggerCacheSingleton(context), - context.scope(), context); + context); } ProtobufTypes::MessagePtr HttpGrpcAccessLogFactory::createEmptyConfigProto() { diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc index 979ef61b059d9..6b5079e86d7fe 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc @@ -28,10 +28,11 @@ HttpGrpcAccessLog::ThreadLocalLogger::ThreadLocalLogger( HttpGrpcAccessLog::HttpGrpcAccessLog( AccessLog::FilterPtr&& filter, envoy::extensions::access_loggers::grpc::v3::HttpGrpcAccessLogConfig config, - ThreadLocal::SlotAllocator& tls, GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache, - Stats::Scope& scope, Server::Configuration::CommonFactoryContext& context) - : Common::ImplBase(std::move(filter)), scope_(scope), config_(std::move(config)), - tls_slot_(tls.allocateSlot()), access_logger_cache_(std::move(access_logger_cache)) { + GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache, + Server::Configuration::CommonFactoryContext& context) + : Common::ImplBase(std::move(filter)), scope_(context.scope()), config_(std::move(config)), + tls_slot_(context.threadLocal().allocateSlot()), + access_logger_cache_(std::move(access_logger_cache)) { for (const auto& header : config_.additional_request_headers_to_log()) { request_headers_to_log_.emplace_back(header); } @@ -50,9 +51,9 @@ HttpGrpcAccessLog::HttpGrpcAccessLog( config_.common_config(), transport_version, Common::GrpcAccessLoggerType::HTTP, scope_)); }); - if (config.has_common_config() && config.common_config().has_buffer_log_filter()) { + if (config_.has_common_config() && config_.common_config().has_buffer_log_filter()) { critical_log_filter_ = AccessLog::FilterFactory::fromProto( - config.common_config().buffer_log_filter(), context.runtime(), + config_.common_config().buffer_log_filter(), context.runtime(), context.api().randomGenerator(), context.messageValidationVisitor()); } } diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h index d56a7c248efe2..ec241c422dcb6 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h @@ -28,9 +28,8 @@ class HttpGrpcAccessLog : public Common::ImplBase { public: HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, envoy::extensions::access_loggers::grpc::v3::HttpGrpcAccessLogConfig config, - ThreadLocal::SlotAllocator& tls, GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache, - Stats::Scope& scope, Server::Configuration::CommonFactoryContext& context); + Server::Configuration::CommonFactoryContext& context); private: /** diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 4a97003153034..011cfd3373ca9 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -70,8 +70,9 @@ class GrpcAccessLoggerImplTest : public testing::Test { : async_client_(new Grpc::MockAsyncClient), timer_(new Event::MockTimer(&dispatcher_)), grpc_access_logger_impl_test_helper_(local_info_, async_client_) { EXPECT_CALL(*timer_, enableTimer(_, _)); + config_.set_log_name("test_log_name"); logger_ = std::make_unique( - Grpc::RawAsyncClientPtr{async_client_}, "test_log_name", FlushInterval, BUFFER_SIZE_BYTES, + Grpc::RawAsyncClientPtr{async_client_}, config_, FlushInterval, BUFFER_SIZE_BYTES, dispatcher_, local_info_, stats_store_, envoy::config::core::v3::ApiVersion::AUTO); } @@ -82,6 +83,7 @@ class GrpcAccessLoggerImplTest : public testing::Test { Event::MockTimer* timer_; std::unique_ptr logger_; GrpcAccessLoggerImplTestHelper grpc_access_logger_impl_test_helper_; + envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig config_; }; TEST_F(GrpcAccessLoggerImplTest, LogHttp) { diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index 5e604f066377b..b05a218623a70 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -71,8 +71,8 @@ class HttpGrpcAccessLogTest : public testing::Test { EXPECT_EQ(Common::GrpcAccessLoggerType::HTTP, logger_type); return logger_; }); - access_log_ = std::make_unique(AccessLog::FilterPtr{filter_}, config_, tls_, - logger_cache_, scope_, factory_context_); + access_log_ = std::make_unique(AccessLog::FilterPtr{filter_}, config_, + logger_cache_, factory_context_); } void expectLog(const std::string& expected_log_entry_yaml) { From 8f51ae2c08cd368a7c1e57227df66db70a73eaf0 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 2 Aug 2021 07:52:19 +0000 Subject: [PATCH 05/39] wip Signed-off-by: Shikugawa --- .../common/grpc_access_logger.h | 8 +- .../grpc/grpc_access_log_impl.cc | 2 +- .../grpc/grpc_access_log_impl.h | 26 +++- test/common/grpc/grpc_client_integration.h | 4 + .../grpc/grpc_access_log_impl_test.cc | 46 +++++++ .../grpc/http_grpc_access_log_impl_test.cc | 98 ++++++++++++-- .../http_grpc_access_log_integration_test.cc | 128 ++++++++++++++++++ 7 files changed, 292 insertions(+), 20 deletions(-) diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index b97aa249eed2d..287cda8885e07 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -153,7 +153,6 @@ template class GrpcAccessLogClient { #define ALL_GRPC_ACCESS_LOGGER_STATS(COUNTER) \ COUNTER(logs_written) \ COUNTER(logs_dropped) -// TODO(shikugawa): implement critical message related stats. /** * Wrapper struct for the access log stats. @see stats_macros.h @@ -222,6 +221,11 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger= max_buffer_size_bytes_) { + flushFatal(); + } + return; } @@ -287,7 +291,7 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerisStreamStarted()) { + if (!fatal_client_->isStreamStarted()) { initFatalMessage(); } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 4168f0bebec9f..8f9f8e438c605 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -35,7 +35,7 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( client, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs"), - dispatcher, PROTOBUF_GET_MS_OR_DEFAULT(config, message_ack_timeout, 5000), + dispatcher, scope, PROTOBUF_GET_MS_OR_DEFAULT(config, message_ack_timeout, 5000), transport_api_version); } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 9ad0922390d3f..e6f594e5a7a03 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -9,6 +9,7 @@ #include "envoy/grpc/async_client_manager.h" #include "envoy/local_info/local_info.h" #include "envoy/service/accesslog/v3/als.pb.h" +#include "envoy/stats/stats_macros.h" #include "envoy/thread_local/thread_local.h" #include "source/extensions/access_loggers/common/grpc_access_logger.h" @@ -18,6 +19,14 @@ namespace Extensions { namespace AccessLoggers { namespace GrpcCommon { +#define CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(COUNTER, GAUGE) \ + COUNTER(fatal_logs_succeeded) \ + GAUGE(pending_fatal_logs, Accumulate) + +struct CriticalAccessLoggerGrpcClientStats { + CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) +}; + class CriticalAccessLoggerGrpcClientImpl : public Common::CriticalAccessLoggerGrpcClient< envoy::service::accesslog::v3::StreamAccessLogsMessage> { @@ -27,15 +36,18 @@ class CriticalAccessLoggerGrpcClientImpl CriticalAccessLoggerGrpcClientImpl(const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& method, - Event::Dispatcher& dispatcher, uint64_t message_ack_timeout) - : CriticalAccessLoggerGrpcClientImpl(client, method, dispatcher, message_ack_timeout, + Event::Dispatcher& dispatcher, Stats::Scope& scope, + uint64_t message_ack_timeout) + : CriticalAccessLoggerGrpcClientImpl(client, method, dispatcher, scope, message_ack_timeout, absl::nullopt) {} CriticalAccessLoggerGrpcClientImpl( const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& method, - Event::Dispatcher& dispatcher, uint64_t message_ack_timeout, + Event::Dispatcher& dispatcher, Stats::Scope& scope, uint64_t message_ack_timeout, absl::optional transport_api_version) : client_(client), service_method_(method), dispatcher_(dispatcher), - message_ack_timeout_(message_ack_timeout), transport_api_version_(transport_api_version) {} + message_ack_timeout_(message_ack_timeout), transport_api_version_(transport_api_version), + stats_({CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(POOL_COUNTER(scope), POOL_GAUGE(scope))}) { + } struct BufferedMessage { enum class State { @@ -65,6 +77,8 @@ class CriticalAccessLoggerGrpcClientImpl switch (message->status()) { case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK: + parent_.stats_.fatal_logs_succeeded_.inc(); + parent_.stats_.pending_fatal_logs_.dec(); parent_.buffered_messages_.erase(id); break; case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK: @@ -75,6 +89,8 @@ class CriticalAccessLoggerGrpcClientImpl default: return; } + + return; } } void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {} @@ -108,6 +124,7 @@ class CriticalAccessLoggerGrpcClientImpl uint32_t id = MessageUtil::hash(message); buffered_messages_[id] = BufferedMessage{nullptr, BufferedMessage::State::Initial, message}; + stats_.pending_fatal_logs_.inc(); for (auto&& buffered_message : buffered_messages_) { uint32_t id = buffered_message.first; @@ -146,6 +163,7 @@ class CriticalAccessLoggerGrpcClientImpl Event::Dispatcher& dispatcher_; std::chrono::milliseconds message_ack_timeout_; const absl::optional transport_api_version_; + CriticalAccessLoggerGrpcClientStats stats_; }; class GrpcAccessLoggerImpl diff --git a/test/common/grpc/grpc_client_integration.h b/test/common/grpc/grpc_client_integration.h index 0c559cf699242..9b0e93edeeedf 100644 --- a/test/common/grpc/grpc_client_integration.h +++ b/test/common/grpc/grpc_client_integration.h @@ -126,6 +126,10 @@ class DeltaSotwIntegrationParamTest testing::Values(envoy::config::core::v3::ApiVersion::V3, \ envoy::config::core::v3::ApiVersion::V2, \ envoy::config::core::v3::ApiVersion::AUTO)) +#define ONLYV3_GRPC_CLIENT_INTEGRATION_PARAMS \ + testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ + testing::ValuesIn(TestEnvironment::getsGrpcVersionsForTest()), \ + testing::Values(envoy::config::core::v3::ApiVersion::V3)) #define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::ValuesIn(TestEnvironment::getsGrpcVersionsForTest()), \ diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 011cfd3373ca9..62fcf65da8bc4 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -59,6 +59,21 @@ class GrpcAccessLoggerImplTestHelper { })); } + void expectStreamCriticalMessage(const std::string& expected_message_yaml) { + envoy::service::accesslog::v3::StreamAccessLogsMessage expected_message; + TestUtility::loadFromYaml(expected_message_yaml, expected_message); + EXPECT_CALL(stream_, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + EXPECT_CALL(stream_, sendMessageRaw_(_, false)) + .WillOnce(Invoke([expected_message](Buffer::InstancePtr& request, bool) { + envoy::service::accesslog::v3::StreamAccessLogsMessage message; + Buffer::ZeroCopyInputStreamImpl request_stream(std::move(request)); + EXPECT_TRUE(message.ParseFromZeroCopyStream(&request_stream)); + EXPECT_GT(message.id(), 0); + message.set_id(0); + EXPECT_EQ(message.DebugString(), expected_message.DebugString()); + })); + } + private: MockAccessLogStream stream_; AccessLogCallbacks* callbacks_; @@ -124,6 +139,37 @@ TEST_F(GrpcAccessLoggerImplTest, LogTcp) { logger_->log(envoy::data::accesslog::v3::TCPAccessLogEntry(tcp_entry), false); } +class CriticalGrpcAccessLoggerImplTest : public GrpcAccessLoggerImplTest { +public: + CriticalGrpcAccessLoggerImplTest() { + mock_buffer_timer_ = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*mock_buffer_timer_, enableTimer(_, _)); + } + +private: + Event::MockTimer* mock_buffer_timer_; +}; + +TEST_F(CriticalGrpcAccessLoggerImplTest, CriticalLogHttp) { + grpc_access_logger_impl_test_helper_.expectStreamCriticalMessage(R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + log_name: test_log_name +http_logs: + log_entry: + request: + path: /test/path1 +id: 0 +)EOF"); + envoy::data::accesslog::v3::HTTPAccessLogEntry entry; + entry.mutable_request()->set_path("/test/path1"); + logger_->log(envoy::data::accesslog::v3::HTTPAccessLogEntry(entry), true); +} + class GrpcAccessLoggerCacheImplTest : public testing::Test { public: GrpcAccessLoggerCacheImplTest() diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index b05a218623a70..0e662fe63dddf 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -1,5 +1,6 @@ #include +#include "envoy/config/accesslog/v3/accesslog.pb.h" #include "envoy/data/accesslog/v3/accesslog.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" @@ -20,6 +21,7 @@ using namespace std::chrono_literals; using testing::_; using testing::An; +using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::NiceMock; @@ -75,14 +77,14 @@ class HttpGrpcAccessLogTest : public testing::Test { logger_cache_, factory_context_); } - void expectLog(const std::string& expected_log_entry_yaml) { + void expectLog(const std::string& expected_log_entry_yaml, bool expect_critical) { if (access_log_ == nullptr) { init(); } HTTPAccessLogEntry expected_log_entry; TestUtility::loadFromYaml(expected_log_entry_yaml, expected_log_entry); - EXPECT_CALL(*logger_, log(An(), _)) + EXPECT_CALL(*logger_, log(An(), Eq(expect_critical))) .WillOnce(Invoke( [expected_log_entry](envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry, bool) { EXPECT_EQ(entry.DebugString(), expected_log_entry.DebugString()); @@ -119,7 +121,8 @@ class HttpGrpcAccessLogTest : public testing::Test { request_headers_bytes: {} response: {{}} )EOF", - request_method, request_method.length() + 7)); + request_method, request_method.length() + 7), + false); access_log_->log(&request_headers, nullptr, nullptr, stream_info); } @@ -200,7 +203,8 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { value: 10s request: {} response: {} -)EOF"); +)EOF", + false); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -230,7 +234,8 @@ response: {} nanos: 2000000 request: {} response: {} -)EOF"); +)EOF", + false); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -332,7 +337,8 @@ protocol_version: HTTP10 response_headers_bytes: 10 response_body_bytes: 20 response_code_details: "via_upstream" -)EOF"); +)EOF", + false); access_log_->log(&request_headers, &response_headers, nullptr, stream_info); } @@ -367,7 +373,8 @@ protocol_version: HTTP10 request_method: "METHOD_UNSPECIFIED" request_headers_bytes: 16 response: {} -)EOF"); +)EOF", + false); access_log_->log(&request_headers, nullptr, nullptr, stream_info); } @@ -433,7 +440,8 @@ response: {} request_method: "METHOD_UNSPECIFIED" request_headers_bytes: 16 response: {} -)EOF"); +)EOF", + false); access_log_->log(&request_headers, nullptr, nullptr, stream_info); } @@ -483,7 +491,8 @@ response: {} request: request_method: "METHOD_UNSPECIFIED" response: {} -)EOF"); +)EOF", + false); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -533,7 +542,8 @@ response: {} request: request_method: "METHOD_UNSPECIFIED" response: {} -)EOF"); +)EOF", + false); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -583,7 +593,8 @@ response: {} request: request_method: "METHOD_UNSPECIFIED" response: {} -)EOF"); +)EOF", + false); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -633,7 +644,8 @@ response: {} request: request_method: "METHOD_UNSPECIFIED" response: {} -)EOF"); +)EOF", + false); access_log_->log(nullptr, nullptr, nullptr, stream_info); } } @@ -720,7 +732,8 @@ TEST_F(HttpGrpcAccessLogTest, MarshallingAdditionalHeaders) { response_trailers: "x-logged-trailer": "value" "x-empty-trailer": "" -)EOF"); +)EOF", + false); access_log_->log(&request_headers, &response_headers, &response_trailers, stream_info); } } @@ -738,6 +751,65 @@ TEST_F(HttpGrpcAccessLogTest, LogWithRequestMethod) { expectLogRequestMethod("PATCH"); } +TEST_F(HttpGrpcAccessLogTest, BufferLogFilterTest) { + const std::string filter_yaml = R"EOF( +status_code_filter: + comparison: + op: EQ + value: + default_value: 200 + runtime_key: access_log.access_error.status + )EOF"; + + envoy::config::accesslog::v3::AccessLogFilter config; + TestUtility::loadFromYaml(filter_yaml, config); + *config_.mutable_common_config()->mutable_buffer_log_filter() = config; + + init(); + + { + NiceMock stream_info; + stream_info.host_ = nullptr; + stream_info.start_time_ = SystemTime(1h); + stream_info.response_code_ = 200; + + Http::TestRequestHeaderMapImpl request_headers{ + {":scheme", "scheme_value"}, + {":authority", "authority_value"}, + {":path", "path_value"}, + {":method", "POST"}, + }; + + expectLog(R"EOF( +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_direct_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: + seconds: 3600 +request: + scheme: "scheme_value" + authority: "authority_value" + path: "path_value" + request_method: "POST" + request_headers_bytes: 70 +response: + response_code: 200 +)EOF", + true); + access_log_->log(&request_headers, nullptr, nullptr, stream_info); + } +} + } // namespace } // namespace HttpGrpc } // namespace AccessLoggers diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index 0699fc5e81f9d..5242b2bdf61c9 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -124,6 +124,134 @@ class AccessLogIntegrationTest : public Grpc::VersionedGrpcClientIntegrationPara FakeStreamPtr access_log_request_; }; +class CriticalAccessLogIntegrationTest : public AccessLogIntegrationTest { +public: + void initialize() override { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* accesslog_cluster = bootstrap.mutable_static_resources()->add_clusters(); + accesslog_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + accesslog_cluster->set_name("accesslog"); + ConfigHelper::setHttp2(*accesslog_cluster); + }); + + config_helper_.addConfigModifier( + [this]( + envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& + hcm) { + auto* access_log = hcm.add_access_log(); + access_log->set_name("grpc_accesslog"); + + envoy::extensions::access_loggers::grpc::v3::HttpGrpcAccessLogConfig config; + auto* common_config = config.mutable_common_config(); + common_config->set_log_name("foo"); + common_config->set_transport_api_version(apiVersion()); + setGrpcService(*common_config->mutable_grpc_service(), "accesslog", + fake_upstreams_.back()->localAddress()); + access_log->mutable_typed_config()->PackFrom(config); + + const std::string filter_yaml = R"EOF( + status_code_filter: + comparison: + op: GE + value: + default_value: 100 + runtime_key: access_log.access_error.status + )EOF"; + + envoy::config::accesslog::v3::AccessLogFilter filter_config; + TestUtility::loadFromYaml(filter_yaml, filter_config); + *common_config->mutable_buffer_log_filter() = filter_config; + }); + + HttpIntegrationTest::initialize(); + } + + ABSL_MUST_USE_RESULT + AssertionResult waitForCriticalAccessLogRequest(const std::string& expected_request_msg_yaml) { + envoy::service::accesslog::v3::StreamAccessLogsMessage request_msg; + VERIFY_ASSERTION(access_log_request_->waitForGrpcMessage(*dispatcher_, request_msg)); + EXPECT_EQ("POST", access_log_request_->headers().getMethodValue()); + EXPECT_EQ("envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs", + access_log_request_->headers().getPathValue()); + EXPECT_EQ("application/grpc", access_log_request_->headers().getContentTypeValue()); + + envoy::service::accesslog::v3::StreamAccessLogsMessage expected_request_msg; + TestUtility::loadFromYaml(expected_request_msg_yaml, expected_request_msg); + + // Clear fields which are not deterministic. + auto* log_entry = request_msg.mutable_http_logs()->mutable_log_entry(0); + log_entry->mutable_common_properties()->clear_downstream_remote_address(); + log_entry->mutable_common_properties()->clear_downstream_direct_remote_address(); + log_entry->mutable_common_properties()->clear_downstream_local_address(); + log_entry->mutable_common_properties()->clear_start_time(); + log_entry->mutable_common_properties()->clear_time_to_last_rx_byte(); + log_entry->mutable_common_properties()->clear_time_to_first_downstream_tx_byte(); + log_entry->mutable_common_properties()->clear_time_to_last_downstream_tx_byte(); + log_entry->mutable_request()->clear_request_id(); + if (request_msg.has_identifier()) { + auto* node = request_msg.mutable_identifier()->mutable_node(); + node->clear_extensions(); + node->clear_user_agent_build_version(); + } + Config::VersionUtil::scrubHiddenEnvoyDeprecated(request_msg); + Config::VersionUtil::scrubHiddenEnvoyDeprecated(expected_request_msg); + EXPECT_THAT(request_msg, ProtoEq(expected_request_msg)); + return AssertionSuccess(); + } + + ABSL_MUST_USE_RESULT + AssertionResult waitForCriticalAccessLogResponse( + envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::Status expected_status) { + envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse resp; + resp.set_status(expected_status); + Buffer::OwnedImpl body(resp.DebugString()); + access_log_request_->encodeData(body, false); + return AssertionSuccess(); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersionsCientType, CriticalAccessLogIntegrationTest, + ONLYV3_GRPC_CLIENT_INTEGRATION_PARAMS, + Grpc::VersionedGrpcClientIntegrationParamTest::protocolTestParamsToString); + +TEST_P(CriticalAccessLogIntegrationTest, BasicFlow) { + XDS_DEPRECATED_FEATURE_TEST_SKIP; + testRouterNotFound(); + ASSERT_TRUE(waitForAccessLogConnection()); + ASSERT_TRUE(waitForAccessLogStream()); + + ASSERT_TRUE(waitForCriticalAccessLogRequest(fmt::format(R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + user_agent_name: "envoy" + log_name: foo +http_logs: + log_entry: + common_properties: + response_flags: + no_route_found: true + protocol_version: HTTP11 + request: + scheme: http + authority: host + path: /notfound + request_headers_bytes: 118 + request_method: GET + response: + response_code: + value: 404 + response_code_details: "route_not_found" + response_headers_bytes: 54 +)EOF"))); + + ASSERT_TRUE(waitForCriticalAccessLogResponse( + envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK)); +} + INSTANTIATE_TEST_SUITE_P(IpVersionsCientType, AccessLogIntegrationTest, VERSIONED_GRPC_CLIENT_INTEGRATION_PARAMS, Grpc::VersionedGrpcClientIntegrationParamTest::protocolTestParamsToString); From 5ed729fb6b459f9243acfe1d3e65abce6a047948 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 2 Aug 2021 11:42:57 +0000 Subject: [PATCH 06/39] test Signed-off-by: Shikugawa --- .../grpc/grpc_access_log_impl.cc | 4 +- .../grpc/grpc_access_log_impl.h | 21 ++- test/extensions/access_loggers/grpc/BUILD | 1 + .../http_grpc_access_log_integration_test.cc | 139 ++++++++++++++++-- 4 files changed, 139 insertions(+), 26 deletions(-) diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 8f9f8e438c605..a8de5d5cfd9c2 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -10,8 +10,6 @@ #include "source/common/config/utility.h" #include "source/common/grpc/typed_async_client.h" -const char GRPC_LOG_STATS_PREFIX[] = "access_logs.grpc_access_log."; - namespace Envoy { namespace Extensions { namespace AccessLoggers { @@ -25,7 +23,7 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( envoy::config::core::v3::ApiVersion transport_api_version) : GrpcAccessLogger( std::move(client), buffer_flush_interval_msec, max_buffer_size_bytes, dispatcher, scope, - GRPC_LOG_STATS_PREFIX, + GRPC_LOG_STATS_PREFIX.data(), Grpc::VersionedMethods("envoy.service.accesslog.v3.AccessLogService.StreamAccessLogs", "envoy.service.accesslog.v2.AccessLogService.StreamAccessLogs") .getMethodDescriptorForVersion(transport_api_version), diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index e6f594e5a7a03..7976b92a62d89 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -19,9 +19,12 @@ namespace Extensions { namespace AccessLoggers { namespace GrpcCommon { +static constexpr absl::string_view GRPC_LOG_STATS_PREFIX = "access_logs.grpc_access_log."; + #define CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(COUNTER, GAUGE) \ - COUNTER(fatal_logs_succeeded) \ - GAUGE(pending_fatal_logs, Accumulate) + COUNTER(critical_logs_succeeded) \ + COUNTER(pending_timeout) \ + GAUGE(pending_critical_logs, Accumulate) struct CriticalAccessLoggerGrpcClientStats { CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) @@ -46,8 +49,9 @@ class CriticalAccessLoggerGrpcClientImpl absl::optional transport_api_version) : client_(client), service_method_(method), dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout), transport_api_version_(transport_api_version), - stats_({CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(POOL_COUNTER(scope), POOL_GAUGE(scope))}) { - } + stats_({CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS( + POOL_COUNTER_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()), + POOL_GAUGE_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()))}) {} struct BufferedMessage { enum class State { @@ -77,8 +81,8 @@ class CriticalAccessLoggerGrpcClientImpl switch (message->status()) { case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK: - parent_.stats_.fatal_logs_succeeded_.inc(); - parent_.stats_.pending_fatal_logs_.dec(); + parent_.stats_.critical_logs_succeeded_.inc(); + parent_.stats_.pending_critical_logs_.dec(); parent_.buffered_messages_.erase(id); break; case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK: @@ -124,15 +128,16 @@ class CriticalAccessLoggerGrpcClientImpl uint32_t id = MessageUtil::hash(message); buffered_messages_[id] = BufferedMessage{nullptr, BufferedMessage::State::Initial, message}; - stats_.pending_fatal_logs_.inc(); + stats_.pending_critical_logs_.inc(); for (auto&& buffered_message : buffered_messages_) { uint32_t id = buffered_message.first; buffered_message.second.message_.set_id(id); buffered_message.second.state_ = BufferedMessage::State::Pending; - buffered_message.second.timer_ = dispatcher_.createTimer([&buffered_message]() { + buffered_message.second.timer_ = dispatcher_.createTimer([&]() { if (buffered_message.second.state_ == BufferedMessage::State::Pending) { + stats_.pending_timeout_.inc(); buffered_message.second.state_ = BufferedMessage::State::Initial; buffered_message.second.timer_->disableTimer(); buffered_message.second.timer_.reset(); diff --git a/test/extensions/access_loggers/grpc/BUILD b/test/extensions/access_loggers/grpc/BUILD index 31616c4a41874..4e5275d31d682 100644 --- a/test/extensions/access_loggers/grpc/BUILD +++ b/test/extensions/access_loggers/grpc/BUILD @@ -56,6 +56,7 @@ envoy_extension_cc_test( "//test/mocks/ssl:ssl_mocks", "//test/mocks/stream_info:stream_info_mocks", "//test/mocks/thread_local:thread_local_mocks", + "@envoy_api//envoy/config/accesslog/v3:pkg_cc_proto", "@envoy_api//envoy/data/accesslog/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/access_loggers/grpc/v3:pkg_cc_proto", ], diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index 5242b2bdf61c9..15df25a257281 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -1,3 +1,5 @@ +#include + #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" @@ -147,8 +149,6 @@ class CriticalAccessLogIntegrationTest : public AccessLogIntegrationTest { common_config->set_transport_api_version(apiVersion()); setGrpcService(*common_config->mutable_grpc_service(), "accesslog", fake_upstreams_.back()->localAddress()); - access_log->mutable_typed_config()->PackFrom(config); - const std::string filter_yaml = R"EOF( status_code_filter: comparison: @@ -161,6 +161,7 @@ class CriticalAccessLogIntegrationTest : public AccessLogIntegrationTest { envoy::config::accesslog::v3::AccessLogFilter filter_config; TestUtility::loadFromYaml(filter_yaml, filter_config); *common_config->mutable_buffer_log_filter() = filter_config; + access_log->mutable_typed_config()->PackFrom(config); }); HttpIntegrationTest::initialize(); @@ -171,7 +172,7 @@ class CriticalAccessLogIntegrationTest : public AccessLogIntegrationTest { envoy::service::accesslog::v3::StreamAccessLogsMessage request_msg; VERIFY_ASSERTION(access_log_request_->waitForGrpcMessage(*dispatcher_, request_msg)); EXPECT_EQ("POST", access_log_request_->headers().getMethodValue()); - EXPECT_EQ("envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs", + EXPECT_EQ("/envoy.service.accesslog.v3.AccessLogService/BufferedCriticalAccessLogs", access_log_request_->headers().getPathValue()); EXPECT_EQ("application/grpc", access_log_request_->headers().getContentTypeValue()); @@ -195,26 +196,21 @@ class CriticalAccessLogIntegrationTest : public AccessLogIntegrationTest { } Config::VersionUtil::scrubHiddenEnvoyDeprecated(request_msg); Config::VersionUtil::scrubHiddenEnvoyDeprecated(expected_request_msg); + EXPECT_GE(request_msg.id(), 0); + pending_message_id_ = request_msg.id(); + request_msg.clear_id(); EXPECT_THAT(request_msg, ProtoEq(expected_request_msg)); return AssertionSuccess(); } - ABSL_MUST_USE_RESULT - AssertionResult waitForCriticalAccessLogResponse( - envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::Status expected_status) { - envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse resp; - resp.set_status(expected_status); - Buffer::OwnedImpl body(resp.DebugString()); - access_log_request_->encodeData(body, false); - return AssertionSuccess(); - } + uint32_t pending_message_id_; }; INSTANTIATE_TEST_SUITE_P(IpVersionsCientType, CriticalAccessLogIntegrationTest, ONLYV3_GRPC_CLIENT_INTEGRATION_PARAMS, Grpc::VersionedGrpcClientIntegrationParamTest::protocolTestParamsToString); -TEST_P(CriticalAccessLogIntegrationTest, BasicFlow) { +TEST_P(CriticalAccessLogIntegrationTest, BasicAckFlow) { XDS_DEPRECATED_FEATURE_TEST_SKIP; testRouterNotFound(); ASSERT_TRUE(waitForAccessLogConnection()); @@ -248,8 +244,121 @@ TEST_P(CriticalAccessLogIntegrationTest, BasicFlow) { response_headers_bytes: 54 )EOF"))); - ASSERT_TRUE(waitForCriticalAccessLogResponse( - envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK)); + access_log_request_->startGrpcStream(); + envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse response_msg; + response_msg.set_id(pending_message_id_); + pending_message_id_ = 0; + response_msg.set_status(envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK); + access_log_request_->sendGrpcMessage(response_msg); + access_log_request_->finishGrpcStream(Grpc::Status::Ok); + switch (clientType()) { + case Grpc::ClientType::EnvoyGrpc: + test_server_->waitForGaugeEq("cluster.accesslog.upstream_rq_active", 0); + break; + case Grpc::ClientType::GoogleGrpc: + test_server_->waitForCounterGe("grpc.accesslog.streams_closed_0", 1); + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + + test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_succeeded", 1); + test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 0); + cleanup(); +} + +TEST_P(CriticalAccessLogIntegrationTest, BasicNackFlow) { + XDS_DEPRECATED_FEATURE_TEST_SKIP; + testRouterNotFound(); + ASSERT_TRUE(waitForAccessLogConnection()); + ASSERT_TRUE(waitForAccessLogStream()); + + ASSERT_TRUE(waitForCriticalAccessLogRequest(fmt::format(R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + user_agent_name: "envoy" + log_name: foo +http_logs: + log_entry: + common_properties: + response_flags: + no_route_found: true + protocol_version: HTTP11 + request: + scheme: http + authority: host + path: /notfound + request_headers_bytes: 118 + request_method: GET + response: + response_code: + value: 404 + response_code_details: "route_not_found" + response_headers_bytes: 54 +)EOF"))); + + access_log_request_->startGrpcStream(); + envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse response_msg; + response_msg.set_id(pending_message_id_); + pending_message_id_ = 0; + response_msg.set_status(envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK); + access_log_request_->sendGrpcMessage(response_msg); + access_log_request_->finishGrpcStream(Grpc::Status::Ok); + switch (clientType()) { + case Grpc::ClientType::EnvoyGrpc: + test_server_->waitForGaugeEq("cluster.accesslog.upstream_rq_active", 0); + break; + case Grpc::ClientType::GoogleGrpc: + test_server_->waitForCounterGe("grpc.accesslog.streams_closed_0", 1); + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + + test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 1); + cleanup(); +} + +TEST_P(CriticalAccessLogIntegrationTest, BasicFlow) { + testRouterNotFound(); + ASSERT_TRUE(waitForAccessLogConnection()); + ASSERT_TRUE(waitForAccessLogStream()); + + ASSERT_TRUE(waitForCriticalAccessLogRequest(fmt::format(R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + user_agent_name: "envoy" + log_name: foo +http_logs: + log_entry: + common_properties: + response_flags: + no_route_found: true + protocol_version: HTTP11 + request: + scheme: http + authority: host + path: /notfound + request_headers_bytes: 118 + request_method: GET + response: + response_code: + value: 404 + response_code_details: "route_not_found" + response_headers_bytes: 54 +)EOF"))); + + test_server_->waitForCounterEq("access_logs.grpc_access_log.pending_timeout", 1); + test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 1); + cleanup(); } INSTANTIATE_TEST_SUITE_P(IpVersionsCientType, AccessLogIntegrationTest, From 6fbaa22b32acfab0126f80552a7ea856db3edcdd Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 2 Aug 2021 12:02:57 +0000 Subject: [PATCH 07/39] tinyfix Signed-off-by: Shikugawa --- .../common/grpc_access_logger.h | 44 +++++++++---------- .../grpc/grpc_access_log_impl.cc | 20 +++++---- .../grpc/grpc_access_log_impl.h | 8 ++-- .../open_telemetry/grpc_access_log_impl.cc | 2 +- .../open_telemetry/grpc_access_log_impl.h | 10 ++--- .../common/grpc_access_logger_test.cc | 32 +++++++------- .../http_grpc_access_log_integration_test.cc | 4 +- 7 files changed, 59 insertions(+), 61 deletions(-) diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index 287cda8885e07..041b435b9c579 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -204,7 +204,7 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerenableTimer(buffer_flush_interval_msec_); })), max_buffer_size_bytes_(max_buffer_size_bytes), @@ -220,10 +220,11 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger= max_buffer_size_bytes_) { - flushFatal(); + if (approximate_critical_message_size_bytes_ >= max_buffer_size_bytes_) { + flushCriticalMessage(); } return; @@ -239,12 +240,7 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger= max_buffer_size_bytes_) { @@ -254,21 +250,21 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger client_; - std::unique_ptr> fatal_client_; + std::unique_ptr> critical_client_; LogRequest message_; - LogRequest fatal_message_; + LogRequest critical_message_; private: virtual bool isEmpty() PURE; - virtual bool isFatalEmpty() PURE; + virtual bool isCriticalMessageEmpty() PURE; virtual void initMessage() PURE; - virtual void initFatalMessage() PURE; + virtual void initCriticalMessage() PURE; virtual void addEntry(HttpLogProto&& entry) PURE; virtual void addEntry(TcpLogProto&& entry) PURE; - virtual void addFatalEntry(HttpLogProto&& entry) PURE; - virtual void addFatalEntry(TcpLogProto&& entry) PURE; + virtual void addCriticalMessageEntry(HttpLogProto&& entry) PURE; + virtual void addCriticalMessageEntry(TcpLogProto&& entry) PURE; virtual void clearMessage() { message_.Clear(); } - virtual void clearFatalMessage() { fatal_message_.Clear(); } + virtual void clearCriticalMessage() { critical_message_.Clear(); } void flush() { if (isEmpty()) { @@ -287,16 +283,17 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerisStreamStarted()) { - initFatalMessage(); + if (!critical_client_->isStreamStarted()) { + initCriticalMessage(); } - fatal_client_->flush(fatal_message_); - clearFatalMessage(); + approximate_critical_message_size_bytes_ = 0; + critical_client_->flush(critical_message_); + clearCriticalMessage(); } bool canLogMore() { @@ -317,6 +314,7 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger( + critical_client_ = std::make_unique( client, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs"), @@ -45,20 +45,22 @@ void GrpcAccessLoggerImpl::addEntry(envoy::data::accesslog::v3::TCPAccessLogEntr message_.mutable_tcp_logs()->mutable_log_entry()->Add(std::move(entry)); } -void GrpcAccessLoggerImpl::addFatalEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { - fatal_message_.mutable_http_logs()->mutable_log_entry()->Add(std::move(entry)); +void GrpcAccessLoggerImpl::addCriticalMessageEntry( + envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { + critical_message_.mutable_http_logs()->mutable_log_entry()->Add(std::move(entry)); } -void GrpcAccessLoggerImpl::addFatalEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) { - fatal_message_.mutable_tcp_logs()->mutable_log_entry()->Add(std::move(entry)); +void GrpcAccessLoggerImpl::addCriticalMessageEntry( + envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) { + critical_message_.mutable_tcp_logs()->mutable_log_entry()->Add(std::move(entry)); } bool GrpcAccessLoggerImpl::isEmpty() { return !message_.has_http_logs() && !message_.has_tcp_logs(); } -bool GrpcAccessLoggerImpl::isFatalEmpty() { - return !fatal_message_.has_http_logs() && !fatal_message_.has_tcp_logs(); +bool GrpcAccessLoggerImpl::isCriticalMessageEmpty() { + return !critical_message_.has_http_logs() && !critical_message_.has_tcp_logs(); } void GrpcAccessLoggerImpl::initMessage() { @@ -67,8 +69,8 @@ void GrpcAccessLoggerImpl::initMessage() { identifier->set_log_name(log_name_); } -void GrpcAccessLoggerImpl::initFatalMessage() { - auto* identifier = fatal_message_.mutable_identifier(); +void GrpcAccessLoggerImpl::initCriticalMessage() { + auto* identifier = critical_message_.mutable_identifier(); *identifier->mutable_node() = local_info_.node(); identifier->set_log_name(log_name_); } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 7976b92a62d89..5b72611beeb1c 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -188,12 +188,12 @@ class GrpcAccessLoggerImpl // Extensions::AccessLoggers::GrpcCommon::GrpcAccessLogger void addEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) override; void addEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) override; - void addFatalEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) override; - void addFatalEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) override; + void addCriticalMessageEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) override; + void addCriticalMessageEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) override; bool isEmpty() override; - bool isFatalEmpty() override; + bool isCriticalMessageEmpty() override; void initMessage() override; - void initFatalMessage() override; + void initCriticalMessage() override; const std::string log_name_; const LocalInfo::LocalInfo& local_info_; diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc index 2f8efd089f806..9c68e7df909a6 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc @@ -65,7 +65,7 @@ void GrpcAccessLoggerImpl::addEntry(opentelemetry::proto::logs::v1::LogRecord&& bool GrpcAccessLoggerImpl::isEmpty() { return root_->logs().empty(); } -bool GrpcAccessLoggerImpl::isFatalEmpty() { +bool GrpcAccessLoggerImpl::isCriticalMessageEmpty() { // Critical message must be empty in OpenTelemetry logger. return true; } diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h index f2f30f20cd736..52dbca16fd1f8 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h @@ -47,14 +47,14 @@ class GrpcAccessLoggerImpl void addEntry(opentelemetry::proto::logs::v1::LogRecord&& entry) override; // Non used addEntry method (the above is used for both TCP and HTTP). void addEntry(ProtobufWkt::Empty&& entry) override { (void)entry; }; - void addFatalEntry(opentelemetry::proto::logs::v1::LogRecord&&) override {} - void addFatalEntry(ProtobufWkt::Empty&&) override{}; + void addCriticalMessageEntry(opentelemetry::proto::logs::v1::LogRecord&&) override {} + void addCriticalMessageEntry(ProtobufWkt::Empty&&) override{}; bool isEmpty() override; - bool isFatalEmpty() override; + bool isCriticalMessageEmpty() override; void initMessage() override; void clearMessage() override; - void clearFatalMessage() override {} - void initFatalMessage() override {} + void clearCriticalMessage() override {} + void initCriticalMessage() override {} opentelemetry::proto::logs::v1::InstrumentationLibraryLogs* root_; }; diff --git a/test/extensions/access_loggers/common/grpc_access_logger_test.cc b/test/extensions/access_loggers/common/grpc_access_logger_test.cc index e4bcb051f33be..e8e559ba21129 100644 --- a/test/extensions/access_loggers/common/grpc_access_logger_test.cc +++ b/test/extensions/access_loggers/common/grpc_access_logger_test.cc @@ -75,14 +75,14 @@ class MockGrpcAccessLoggerImpl 1); } - void mockAddFatalEntry(const std::string& key) { - if (!fatal_message_.fields().contains(key)) { + void mockaddCriticalMessageEntry(const std::string& key) { + if (!critical_message_.fields().contains(key)) { ProtobufWkt::Value default_value; default_value.set_number_value(0); - fatal_message_.mutable_fields()->insert({key, default_value}); + critical_message_.mutable_fields()->insert({key, default_value}); } - fatal_message_.mutable_fields()->at(key).set_number_value( - fatal_message_.fields().at(key).number_value() + 1); + critical_message_.mutable_fields()->at(key).set_number_value( + critical_message_.fields().at(key).number_value() + 1); } // Extensions::AccessLoggers::GrpcCommon::GrpcAccessLogger @@ -100,19 +100,19 @@ class MockGrpcAccessLoggerImpl mockAddEntry(MOCK_TCP_LOG_FIELD_NAME); } - void addFatalEntry(ProtobufWkt::Struct&& entry) override { + void addCriticalMessageEntry(ProtobufWkt::Struct&& entry) override { (void)entry; - mockAddFatalEntry(MOCK_HTTP_LOG_FIELD_NAME); + mockaddCriticalMessageEntry(MOCK_HTTP_LOG_FIELD_NAME); } - void addFatalEntry(ProtobufWkt::Empty&& entry) override { + void addCriticalMessageEntry(ProtobufWkt::Empty&& entry) override { (void)entry; - mockAddFatalEntry(MOCK_TCP_LOG_FIELD_NAME); + mockaddCriticalMessageEntry(MOCK_TCP_LOG_FIELD_NAME); } bool isEmpty() override { return message_.fields().empty(); } - bool isFatalEmpty() override { return fatal_message_.fields().empty(); } - void initFatalMessage() override { ++num_fatal_inits_; } + bool isCriticalMessageEmpty() override { return critical_message_.fields().empty(); } + void initCriticalMessage() override { ++num_critical_inits_; } void initMessage() override { ++num_inits_; } void clearMessage() override { @@ -120,15 +120,15 @@ class MockGrpcAccessLoggerImpl num_clears_++; } - void clearFatalMessage() override { - fatal_message_.Clear(); - num_fatal_clears_++; + void clearCriticalMessage() override { + critical_message_.Clear(); + num_critical_clears_++; } int num_inits_ = 0; - int num_fatal_inits_ = 0; + int num_critical_inits_ = 0; int num_clears_ = 0; - int num_fatal_clears_ = 0; + int num_critical_clears_ = 0; }; class GrpcAccessLogTest : public testing::Test { diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index 15df25a257281..20d2d9c2c6e22 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -1,5 +1,3 @@ -#include - #include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" @@ -323,7 +321,7 @@ TEST_P(CriticalAccessLogIntegrationTest, BasicNackFlow) { cleanup(); } -TEST_P(CriticalAccessLogIntegrationTest, BasicFlow) { +TEST_P(CriticalAccessLogIntegrationTest, NoResponseFlow) { testRouterNotFound(); ASSERT_TRUE(waitForAccessLogConnection()); ASSERT_TRUE(waitForAccessLogStream()); From 4110cc79cc30ba87856f33b85140a49bbf3e8511 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 4 Aug 2021 06:34:28 +0000 Subject: [PATCH 08/39] fix api Signed-off-by: Shikugawa --- api/envoy/service/accesslog/v3/als.proto | 13 +- api/envoy/service/accesslog/v4alpha/als.proto | 16 +- .../envoy/service/accesslog/v3/als.proto | 13 +- .../envoy/service/accesslog/v4alpha/als.proto | 16 +- source/extensions/access_loggers/common/BUILD | 1 - .../common/grpc_access_logger.h | 7 +- .../grpc/grpc_access_log_impl.cc | 11 +- .../grpc/grpc_access_log_impl.h | 15 +- .../open_telemetry/grpc_access_log_impl.h | 3 +- .../grpc/grpc_access_log_impl_test.cc | 27 +-- .../http_grpc_access_log_integration_test.cc | 163 +++++++++--------- 11 files changed, 168 insertions(+), 117 deletions(-) diff --git a/api/envoy/service/accesslog/v3/als.proto b/api/envoy/service/accesslog/v3/als.proto index ff25f185aedf7..a07e83be0179a 100644 --- a/api/envoy/service/accesslog/v3/als.proto +++ b/api/envoy/service/accesslog/v3/als.proto @@ -28,7 +28,7 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - rpc BufferedCriticalAccessLogs(stream StreamAccessLogsMessage) + rpc BufferedCriticalAccessLogs(stream BufferedCriticalAccessLogsMessage) returns (stream BufferedCriticalAccessLogsResponse) { } } @@ -105,8 +105,17 @@ message StreamAccessLogsMessage { TCPAccessLogEntries tcp_logs = 3; } +} + +// Stream message for the BufferedCriticalAccessLogs API. +// Envoy opens a stream to the server and streams the access log, +// expecting a response. Each message sent is assigned an individual ID, +// and the state of the message is tracked based on the ID. +message BufferedCriticalAccessLogsMessage { + // The body of the log message sent to BufferedCriticalAccessLogs. + StreamAccessLogsMessage message = 1; // This is an ID to identify the message, and can be added to the Critical Endpoint // response message to uniquely identify the message being buffered in the Envoy. - uint32 id = 4; + uint64 id = 4; } diff --git a/api/envoy/service/accesslog/v4alpha/als.proto b/api/envoy/service/accesslog/v4alpha/als.proto index e50f9a96b1e96..32d09f6b6bfcf 100644 --- a/api/envoy/service/accesslog/v4alpha/als.proto +++ b/api/envoy/service/accesslog/v4alpha/als.proto @@ -28,7 +28,7 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - rpc BufferedCriticalAccessLogs(stream StreamAccessLogsMessage) + rpc BufferedCriticalAccessLogs(stream BufferedCriticalAccessLogsMessage) returns (stream BufferedCriticalAccessLogsResponse) { } } @@ -108,8 +108,20 @@ message StreamAccessLogsMessage { TCPAccessLogEntries tcp_logs = 3; } +} + +// Stream message for the BufferedCriticalAccessLogs API. +// Envoy opens a stream to the server and streams the access log, +// expecting a response. Each message sent is assigned an individual ID, +// and the state of the message is tracked based on the ID. +message BufferedCriticalAccessLogsMessage { + option (udpa.annotations.versioning).previous_message_type = + "envoy.service.accesslog.v3.BufferedCriticalAccessLogsMessage"; + + // The body of the log message sent to BufferedCriticalAccessLogs. + StreamAccessLogsMessage message = 1; // This is an ID to identify the message, and can be added to the Critical Endpoint // response message to uniquely identify the message being buffered in the Envoy. - uint32 id = 4; + uint64 id = 4; } diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto index ff25f185aedf7..a07e83be0179a 100644 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v3/als.proto @@ -28,7 +28,7 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - rpc BufferedCriticalAccessLogs(stream StreamAccessLogsMessage) + rpc BufferedCriticalAccessLogs(stream BufferedCriticalAccessLogsMessage) returns (stream BufferedCriticalAccessLogsResponse) { } } @@ -105,8 +105,17 @@ message StreamAccessLogsMessage { TCPAccessLogEntries tcp_logs = 3; } +} + +// Stream message for the BufferedCriticalAccessLogs API. +// Envoy opens a stream to the server and streams the access log, +// expecting a response. Each message sent is assigned an individual ID, +// and the state of the message is tracked based on the ID. +message BufferedCriticalAccessLogsMessage { + // The body of the log message sent to BufferedCriticalAccessLogs. + StreamAccessLogsMessage message = 1; // This is an ID to identify the message, and can be added to the Critical Endpoint // response message to uniquely identify the message being buffered in the Envoy. - uint32 id = 4; + uint64 id = 4; } diff --git a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto index e50f9a96b1e96..32d09f6b6bfcf 100644 --- a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto @@ -28,7 +28,7 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - rpc BufferedCriticalAccessLogs(stream StreamAccessLogsMessage) + rpc BufferedCriticalAccessLogs(stream BufferedCriticalAccessLogsMessage) returns (stream BufferedCriticalAccessLogsResponse) { } } @@ -108,8 +108,20 @@ message StreamAccessLogsMessage { TCPAccessLogEntries tcp_logs = 3; } +} + +// Stream message for the BufferedCriticalAccessLogs API. +// Envoy opens a stream to the server and streams the access log, +// expecting a response. Each message sent is assigned an individual ID, +// and the state of the message is tracked based on the ID. +message BufferedCriticalAccessLogsMessage { + option (udpa.annotations.versioning).previous_message_type = + "envoy.service.accesslog.v3.BufferedCriticalAccessLogsMessage"; + + // The body of the log message sent to BufferedCriticalAccessLogs. + StreamAccessLogsMessage message = 1; // This is an ID to identify the message, and can be added to the Critical Endpoint // response message to uniquely identify the message being buffered in the Envoy. - uint32 id = 4; + uint64 id = 4; } diff --git a/source/extensions/access_loggers/common/BUILD b/source/extensions/access_loggers/common/BUILD index 2033bc2dd2cd1..bf4c5d10669bf 100644 --- a/source/extensions/access_loggers/common/BUILD +++ b/source/extensions/access_loggers/common/BUILD @@ -59,7 +59,6 @@ envoy_cc_library( "//source/common/protobuf:utility_lib", "@com_google_absl//absl/types:optional", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/access_loggers/grpc/v3:pkg_cc_proto", "@envoy_api//envoy/service/accesslog/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index 041b435b9c579..e178fc634628e 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -182,7 +182,8 @@ template class CriticalAccessLoggerGrpcClient { * The log entries and messages are distinct types to support batching of multiple access log * entries in a single gRPC messages that go on the wire. */ -template +template class GrpcAccessLogger : public Detail::GrpcAccessLogger { public: using Interface = Detail::GrpcAccessLogger; @@ -250,9 +251,9 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger client_; - std::unique_ptr> critical_client_; + std::unique_ptr> critical_client_; LogRequest message_; - LogRequest critical_message_; + CriticalLogRequest critical_message_; private: virtual bool isEmpty() PURE; diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 66cabafe84ae4..4c7116943a46b 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -47,12 +47,14 @@ void GrpcAccessLoggerImpl::addEntry(envoy::data::accesslog::v3::TCPAccessLogEntr void GrpcAccessLoggerImpl::addCriticalMessageEntry( envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { - critical_message_.mutable_http_logs()->mutable_log_entry()->Add(std::move(entry)); + critical_message_.mutable_message()->mutable_http_logs()->mutable_log_entry()->Add( + std::move(entry)); } void GrpcAccessLoggerImpl::addCriticalMessageEntry( envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) { - critical_message_.mutable_tcp_logs()->mutable_log_entry()->Add(std::move(entry)); + critical_message_.mutable_message()->mutable_tcp_logs()->mutable_log_entry()->Add( + std::move(entry)); } bool GrpcAccessLoggerImpl::isEmpty() { @@ -60,7 +62,8 @@ bool GrpcAccessLoggerImpl::isEmpty() { } bool GrpcAccessLoggerImpl::isCriticalMessageEmpty() { - return !critical_message_.has_http_logs() && !critical_message_.has_tcp_logs(); + return !critical_message_.message().has_http_logs() && + !critical_message_.message().has_tcp_logs(); } void GrpcAccessLoggerImpl::initMessage() { @@ -70,7 +73,7 @@ void GrpcAccessLoggerImpl::initMessage() { } void GrpcAccessLoggerImpl::initCriticalMessage() { - auto* identifier = critical_message_.mutable_identifier(); + auto* identifier = critical_message_.mutable_message()->mutable_identifier(); *identifier->mutable_node() = local_info_.node(); identifier->set_log_name(log_name_); } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 5b72611beeb1c..a572b5f9a0863 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -32,9 +32,9 @@ struct CriticalAccessLoggerGrpcClientStats { class CriticalAccessLoggerGrpcClientImpl : public Common::CriticalAccessLoggerGrpcClient< - envoy::service::accesslog::v3::StreamAccessLogsMessage> { + envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage> { public: - using RequestType = envoy::service::accesslog::v3::StreamAccessLogsMessage; + using RequestType = envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage; using ResponseType = envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse; CriticalAccessLoggerGrpcClientImpl(const Grpc::RawAsyncClientSharedPtr& client, @@ -171,11 +171,12 @@ class CriticalAccessLoggerGrpcClientImpl CriticalAccessLoggerGrpcClientStats stats_; }; -class GrpcAccessLoggerImpl - : public Common::GrpcAccessLogger { +class GrpcAccessLoggerImpl : public Common::GrpcAccessLogger< + envoy::data::accesslog::v3::HTTPAccessLogEntry, + envoy::data::accesslog::v3::TCPAccessLogEntry, + envoy::service::accesslog::v3::StreamAccessLogsMessage, + envoy::service::accesslog::v3::StreamAccessLogsResponse, + envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage> { public: GrpcAccessLoggerImpl( const Grpc::RawAsyncClientSharedPtr& client, diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h index 52dbca16fd1f8..b83d6d7996dd5 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h @@ -33,7 +33,8 @@ class GrpcAccessLoggerImpl // as an empty placeholder for the non-used addEntry method. // TODO(itamarkam): Don't cache OpenTelemetry loggers by type (HTTP/TCP). ProtobufWkt::Empty, opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest, - opentelemetry::proto::collector::logs::v1::ExportLogsServiceResponse> { + opentelemetry::proto::collector::logs::v1::ExportLogsServiceResponse, + ProtobufWkt::Empty> { public: GrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, std::string log_name, std::chrono::milliseconds buffer_flush_interval_msec, diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 62fcf65da8bc4..503bd61a204ac 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -60,12 +60,12 @@ class GrpcAccessLoggerImplTestHelper { } void expectStreamCriticalMessage(const std::string& expected_message_yaml) { - envoy::service::accesslog::v3::StreamAccessLogsMessage expected_message; + envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage expected_message; TestUtility::loadFromYaml(expected_message_yaml, expected_message); EXPECT_CALL(stream_, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); EXPECT_CALL(stream_, sendMessageRaw_(_, false)) .WillOnce(Invoke([expected_message](Buffer::InstancePtr& request, bool) { - envoy::service::accesslog::v3::StreamAccessLogsMessage message; + envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage message; Buffer::ZeroCopyInputStreamImpl request_stream(std::move(request)); EXPECT_TRUE(message.ParseFromZeroCopyStream(&request_stream)); EXPECT_GT(message.id(), 0); @@ -152,17 +152,18 @@ class CriticalGrpcAccessLoggerImplTest : public GrpcAccessLoggerImplTest { TEST_F(CriticalGrpcAccessLoggerImplTest, CriticalLogHttp) { grpc_access_logger_impl_test_helper_.expectStreamCriticalMessage(R"EOF( -identifier: - node: - id: node_name - cluster: cluster_name - locality: - zone: zone_name - log_name: test_log_name -http_logs: - log_entry: - request: - path: /test/path1 +message: + identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + log_name: test_log_name + http_logs: + log_entry: + request: + path: /test/path1 id: 0 )EOF"); envoy::data::accesslog::v3::HTTPAccessLogEntry entry; diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index 20d2d9c2c6e22..1e7fdfc7269d5 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -167,18 +167,18 @@ class CriticalAccessLogIntegrationTest : public AccessLogIntegrationTest { ABSL_MUST_USE_RESULT AssertionResult waitForCriticalAccessLogRequest(const std::string& expected_request_msg_yaml) { - envoy::service::accesslog::v3::StreamAccessLogsMessage request_msg; + envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage request_msg; VERIFY_ASSERTION(access_log_request_->waitForGrpcMessage(*dispatcher_, request_msg)); EXPECT_EQ("POST", access_log_request_->headers().getMethodValue()); EXPECT_EQ("/envoy.service.accesslog.v3.AccessLogService/BufferedCriticalAccessLogs", access_log_request_->headers().getPathValue()); EXPECT_EQ("application/grpc", access_log_request_->headers().getContentTypeValue()); - envoy::service::accesslog::v3::StreamAccessLogsMessage expected_request_msg; + envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage expected_request_msg; TestUtility::loadFromYaml(expected_request_msg_yaml, expected_request_msg); // Clear fields which are not deterministic. - auto* log_entry = request_msg.mutable_http_logs()->mutable_log_entry(0); + auto* log_entry = request_msg.mutable_message()->mutable_http_logs()->mutable_log_entry(0); log_entry->mutable_common_properties()->clear_downstream_remote_address(); log_entry->mutable_common_properties()->clear_downstream_direct_remote_address(); log_entry->mutable_common_properties()->clear_downstream_local_address(); @@ -187,8 +187,8 @@ class CriticalAccessLogIntegrationTest : public AccessLogIntegrationTest { log_entry->mutable_common_properties()->clear_time_to_first_downstream_tx_byte(); log_entry->mutable_common_properties()->clear_time_to_last_downstream_tx_byte(); log_entry->mutable_request()->clear_request_id(); - if (request_msg.has_identifier()) { - auto* node = request_msg.mutable_identifier()->mutable_node(); + if (request_msg.message().has_identifier()) { + auto* node = request_msg.mutable_message()->mutable_identifier()->mutable_node(); node->clear_extensions(); node->clear_user_agent_build_version(); } @@ -215,31 +215,32 @@ TEST_P(CriticalAccessLogIntegrationTest, BasicAckFlow) { ASSERT_TRUE(waitForAccessLogStream()); ASSERT_TRUE(waitForCriticalAccessLogRequest(fmt::format(R"EOF( -identifier: - node: - id: node_name - cluster: cluster_name - locality: - zone: zone_name - user_agent_name: "envoy" - log_name: foo -http_logs: - log_entry: - common_properties: - response_flags: - no_route_found: true - protocol_version: HTTP11 - request: - scheme: http - authority: host - path: /notfound - request_headers_bytes: 118 - request_method: GET - response: - response_code: - value: 404 - response_code_details: "route_not_found" - response_headers_bytes: 54 +message: + identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + user_agent_name: "envoy" + log_name: foo + http_logs: + log_entry: + common_properties: + response_flags: + no_route_found: true + protocol_version: HTTP11 + request: + scheme: http + authority: host + path: /notfound + request_headers_bytes: 118 + request_method: GET + response: + response_code: + value: 404 + response_code_details: "route_not_found" + response_headers_bytes: 54 )EOF"))); access_log_request_->startGrpcStream(); @@ -272,31 +273,32 @@ TEST_P(CriticalAccessLogIntegrationTest, BasicNackFlow) { ASSERT_TRUE(waitForAccessLogStream()); ASSERT_TRUE(waitForCriticalAccessLogRequest(fmt::format(R"EOF( -identifier: - node: - id: node_name - cluster: cluster_name - locality: - zone: zone_name - user_agent_name: "envoy" - log_name: foo -http_logs: - log_entry: - common_properties: - response_flags: - no_route_found: true - protocol_version: HTTP11 - request: - scheme: http - authority: host - path: /notfound - request_headers_bytes: 118 - request_method: GET - response: - response_code: - value: 404 - response_code_details: "route_not_found" - response_headers_bytes: 54 +message: + identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + user_agent_name: "envoy" + log_name: foo + http_logs: + log_entry: + common_properties: + response_flags: + no_route_found: true + protocol_version: HTTP11 + request: + scheme: http + authority: host + path: /notfound + request_headers_bytes: 118 + request_method: GET + response: + response_code: + value: 404 + response_code_details: "route_not_found" + response_headers_bytes: 54 )EOF"))); access_log_request_->startGrpcStream(); @@ -327,31 +329,32 @@ TEST_P(CriticalAccessLogIntegrationTest, NoResponseFlow) { ASSERT_TRUE(waitForAccessLogStream()); ASSERT_TRUE(waitForCriticalAccessLogRequest(fmt::format(R"EOF( -identifier: - node: - id: node_name - cluster: cluster_name - locality: - zone: zone_name - user_agent_name: "envoy" - log_name: foo -http_logs: - log_entry: - common_properties: - response_flags: - no_route_found: true - protocol_version: HTTP11 - request: - scheme: http - authority: host - path: /notfound - request_headers_bytes: 118 - request_method: GET - response: - response_code: - value: 404 - response_code_details: "route_not_found" - response_headers_bytes: 54 +message: + identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + user_agent_name: "envoy" + log_name: foo + http_logs: + log_entry: + common_properties: + response_flags: + no_route_found: true + protocol_version: HTTP11 + request: + scheme: http + authority: host + path: /notfound + request_headers_bytes: 118 + request_method: GET + response: + response_code: + value: 404 + response_code_details: "route_not_found" + response_headers_bytes: 54 )EOF"))); test_server_->waitForCounterEq("access_logs.grpc_access_log.pending_timeout", 1); From c1d448a0f9f20af550d0923daabbae20c223b364 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 4 Aug 2021 10:00:28 +0000 Subject: [PATCH 09/39] add buffer limit Signed-off-by: Shikugawa --- .../access_loggers/grpc/v3/als.proto | 7 +++- .../access_loggers/grpc/v4alpha/als.proto | 7 +++- .../access_loggers/grpc/v3/als.proto | 7 +++- .../access_loggers/grpc/v4alpha/als.proto | 7 +++- .../grpc/grpc_access_log_impl.cc | 1 + .../grpc/grpc_access_log_impl.h | 24 +++++++++--- .../grpc/grpc_access_log_impl_test.cc | 38 +++++++++++++++++++ 7 files changed, 82 insertions(+), 9 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index e9dda14622265..5979df747ceec 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -55,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 9] +// [#next-free-field: 10] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.accesslog.v2.CommonGrpcAccessLogConfig"; @@ -106,4 +106,9 @@ message CommonGrpcAccessLogConfig { // the message is considered undeliverable and the failed transmission is buffered again. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; + + // Soft size limit (in bytes) of the buffer used to store messages during processing in a + // critical logger. A critical logger buffers messages until it receives an ACK from upstream. + // The default is 16384. + google.protobuf.UInt32Value pending_critical_buffer_size_bytes = 9; } diff --git a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 58a0c937ee1a5..94bcc337286cd 100644 --- a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -55,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 9] +// [#next-free-field: 10] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.access_loggers.grpc.v3.CommonGrpcAccessLogConfig"; @@ -106,4 +106,9 @@ message CommonGrpcAccessLogConfig { // the message is considered undeliverable and the failed transmission is buffered again. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; + + // Soft size limit (in bytes) of the buffer used to store messages during processing in a + // critical logger. A critical logger buffers messages until it receives an ACK from upstream. + // The default is 16384. + google.protobuf.UInt32Value pending_critical_buffer_size_bytes = 9; } diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index e9dda14622265..5979df747ceec 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -55,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 9] +// [#next-free-field: 10] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.accesslog.v2.CommonGrpcAccessLogConfig"; @@ -106,4 +106,9 @@ message CommonGrpcAccessLogConfig { // the message is considered undeliverable and the failed transmission is buffered again. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; + + // Soft size limit (in bytes) of the buffer used to store messages during processing in a + // critical logger. A critical logger buffers messages until it receives an ACK from upstream. + // The default is 16384. + google.protobuf.UInt32Value pending_critical_buffer_size_bytes = 9; } diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 58a0c937ee1a5..94bcc337286cd 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -55,7 +55,7 @@ message TcpGrpcAccessLogConfig { } // Common configuration for gRPC access logs. -// [#next-free-field: 9] +// [#next-free-field: 10] message CommonGrpcAccessLogConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.extensions.access_loggers.grpc.v3.CommonGrpcAccessLogConfig"; @@ -106,4 +106,9 @@ message CommonGrpcAccessLogConfig { // the message is considered undeliverable and the failed transmission is buffered again. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; + + // Soft size limit (in bytes) of the buffer used to store messages during processing in a + // critical logger. A critical logger buffers messages until it receives an ACK from upstream. + // The default is 16384. + google.protobuf.UInt32Value pending_critical_buffer_size_bytes = 9; } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 4c7116943a46b..156c8b4ce2a04 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -34,6 +34,7 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs"), dispatcher, scope, PROTOBUF_GET_MS_OR_DEFAULT(config, message_ack_timeout, 5000), + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, pending_critical_buffer_size_bytes, 16384), transport_api_version); } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index a572b5f9a0863..8b337a160fc1e 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -23,6 +23,7 @@ static constexpr absl::string_view GRPC_LOG_STATS_PREFIX = "access_logs.grpc_acc #define CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(COUNTER, GAUGE) \ COUNTER(critical_logs_succeeded) \ + COUNTER(ciritcal_logs_disposed) \ COUNTER(pending_timeout) \ GAUGE(pending_critical_logs, Accumulate) @@ -40,18 +41,21 @@ class CriticalAccessLoggerGrpcClientImpl CriticalAccessLoggerGrpcClientImpl(const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& method, Event::Dispatcher& dispatcher, Stats::Scope& scope, - uint64_t message_ack_timeout) + uint64_t message_ack_timeout, + uint64_t max_critical_buffer_size_bytes) : CriticalAccessLoggerGrpcClientImpl(client, method, dispatcher, scope, message_ack_timeout, - absl::nullopt) {} + max_critical_buffer_size_bytes, absl::nullopt) {} CriticalAccessLoggerGrpcClientImpl( const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& method, Event::Dispatcher& dispatcher, Stats::Scope& scope, uint64_t message_ack_timeout, + uint64_t max_critical_buffer_size_bytes, absl::optional transport_api_version) : client_(client), service_method_(method), dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout), transport_api_version_(transport_api_version), stats_({CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS( POOL_COUNTER_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()), - POOL_GAUGE_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()))}) {} + POOL_GAUGE_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()))}), + max_critical_buffer_size_bytes_(max_critical_buffer_size_bytes) {} struct BufferedMessage { enum class State { @@ -83,12 +87,12 @@ class CriticalAccessLoggerGrpcClientImpl case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK: parent_.stats_.critical_logs_succeeded_.inc(); parent_.stats_.pending_critical_logs_.dec(); + parent_.current_critical_buffer_size_bytes_ -= + parent_.buffered_messages_.at(id).message_.ByteSizeLong(); parent_.buffered_messages_.erase(id); break; case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK: parent_.buffered_messages_.at(id).state_ = BufferedMessage::State::Initial; - // TODO(shikugawa): After many NACK occurred, Buffer will be overwhelmed. - // To resolve this, Buffer should have size limit. break; default: return; @@ -127,7 +131,15 @@ class CriticalAccessLoggerGrpcClientImpl } uint32_t id = MessageUtil::hash(message); + const auto message_byte_size = message.ByteSizeLong(); + + if (current_critical_buffer_size_bytes_ + message_byte_size > max_critical_buffer_size_bytes_) { + stats_.ciritcal_logs_disposed_.inc(); + return; + } + buffered_messages_[id] = BufferedMessage{nullptr, BufferedMessage::State::Initial, message}; + current_critical_buffer_size_bytes_ += message_byte_size; stats_.pending_critical_logs_.inc(); for (auto&& buffered_message : buffered_messages_) { @@ -169,6 +181,8 @@ class CriticalAccessLoggerGrpcClientImpl std::chrono::milliseconds message_ack_timeout_; const absl::optional transport_api_version_; CriticalAccessLoggerGrpcClientStats stats_; + uint64_t current_critical_buffer_size_bytes_ = 0; + const uint64_t max_critical_buffer_size_bytes_; }; class GrpcAccessLoggerImpl : public Common::GrpcAccessLogger< diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 503bd61a204ac..9e5f3c3ef88ad 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -74,6 +74,10 @@ class GrpcAccessLoggerImplTestHelper { })); } + void expectStreamCriticalNoMessage() { + EXPECT_CALL(stream_, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + } + private: MockAccessLogStream stream_; AccessLogCallbacks* callbacks_; @@ -171,6 +175,40 @@ id: 0 logger_->log(envoy::data::accesslog::v3::HTTPAccessLogEntry(entry), true); } +class CriticalGrpcAccessLoggerImplBufferLimitTest : public testing::Test { +public: + CriticalGrpcAccessLoggerImplBufferLimitTest() + : async_client_(new Grpc::MockAsyncClient), timer_(new Event::MockTimer(&dispatcher_)), + grpc_access_logger_impl_test_helper_(local_info_, async_client_) { + EXPECT_CALL(*timer_, enableTimer(_, _)); + config_.set_log_name("test_log_name"); + config_.mutable_pending_critical_buffer_size_bytes()->set_value(0); + logger_ = std::make_unique( + Grpc::RawAsyncClientPtr{async_client_}, config_, FlushInterval, BUFFER_SIZE_BYTES, + dispatcher_, local_info_, stats_store_, envoy::config::core::v3::ApiVersion::AUTO); + } + + Grpc::MockAsyncClient* async_client_; + Stats::IsolatedStoreImpl stats_store_; + LocalInfo::MockLocalInfo local_info_; + Event::MockDispatcher dispatcher_; + Event::MockTimer* timer_; + std::unique_ptr logger_; + GrpcAccessLoggerImplTestHelper grpc_access_logger_impl_test_helper_; + envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig config_; +}; + +TEST_F(CriticalGrpcAccessLoggerImplBufferLimitTest, BasicBehavior) { + grpc_access_logger_impl_test_helper_.expectStreamCriticalNoMessage(); + + envoy::data::accesslog::v3::HTTPAccessLogEntry entry; + entry.mutable_request()->set_path("/test/path1"); + logger_->log(envoy::data::accesslog::v3::HTTPAccessLogEntry(entry), true); + EXPECT_EQ( + stats_store_.counterFromString("access_logs.grpc_access_log.ciritcal_logs_disposed").value(), + 1); +} + class GrpcAccessLoggerCacheImplTest : public testing::Test { public: GrpcAccessLoggerCacheImplTest() From 69773e44cc90cb93690f272dcbcccdc87453883c Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 4 Aug 2021 11:14:21 +0000 Subject: [PATCH 10/39] fix Signed-off-by: Shikugawa --- .../extensions/access_loggers/common/grpc_access_logger_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions/access_loggers/common/grpc_access_logger_test.cc b/test/extensions/access_loggers/common/grpc_access_logger_test.cc index e8e559ba21129..6063db35ccb35 100644 --- a/test/extensions/access_loggers/common/grpc_access_logger_test.cc +++ b/test/extensions/access_loggers/common/grpc_access_logger_test.cc @@ -48,7 +48,7 @@ const Protobuf::MethodDescriptor& mockMethodDescriptor() { // standard Struct and Empty protos. class MockGrpcAccessLoggerImpl : public Common::GrpcAccessLogger { + ProtobufWkt::Struct, ProtobufWkt::Struct> { public: MockGrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, std::chrono::milliseconds buffer_flush_interval_msec, From 35dd18794c040b73b25c0013f68713a21d71480d Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 4 Aug 2021 13:57:33 +0000 Subject: [PATCH 11/39] refactor Signed-off-by: Shikugawa --- .../common/grpc_access_logger.h | 38 ++++--------------- .../grpc/grpc_access_log_impl.cc | 30 ++++++++++++++- .../grpc/grpc_access_log_impl.h | 38 +++++++++++-------- .../open_telemetry/grpc_access_log_impl.cc | 5 --- .../open_telemetry/grpc_access_log_impl.h | 8 +--- 5 files changed, 60 insertions(+), 59 deletions(-) diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index e178fc634628e..a5bdcc6b9506f 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -176,14 +176,17 @@ template class CriticalAccessLoggerGrpcClient { virtual bool isStreamStarted() PURE; }; +template +using CriticalAccessLoggerGrpcClientPtr = + std::unique_ptr>; + /** * Base class for defining a gRPC logger with the `HttpLogProto` and `TcpLogProto` access log * entries and `LogRequest` and `LogResponse` gRPC messages. * The log entries and messages are distinct types to support batching of multiple access log * entries in a single gRPC messages that go on the wire. */ -template +template class GrpcAccessLogger : public Detail::GrpcAccessLogger { public: using Interface = Detail::GrpcAccessLogger; @@ -221,13 +224,7 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger= max_buffer_size_bytes_) { - flushCriticalMessage(); - } - + logCritical(std::move(entry)); return; } @@ -251,21 +248,16 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger client_; - std::unique_ptr> critical_client_; LogRequest message_; - CriticalLogRequest critical_message_; private: virtual bool isEmpty() PURE; - virtual bool isCriticalMessageEmpty() PURE; virtual void initMessage() PURE; - virtual void initCriticalMessage() PURE; virtual void addEntry(HttpLogProto&& entry) PURE; virtual void addEntry(TcpLogProto&& entry) PURE; - virtual void addCriticalMessageEntry(HttpLogProto&& entry) PURE; - virtual void addCriticalMessageEntry(TcpLogProto&& entry) PURE; virtual void clearMessage() { message_.Clear(); } - virtual void clearCriticalMessage() { critical_message_.Clear(); } + virtual void flushCriticalMessage() {} + virtual void logCritical(HttpLogProto&&) {} void flush() { if (isEmpty()) { @@ -284,19 +276,6 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerisStreamStarted()) { - initCriticalMessage(); - } - - approximate_critical_message_size_bytes_ = 0; - critical_client_->flush(critical_message_); - clearCriticalMessage(); - } - bool canLogMore() { if (max_buffer_size_bytes_ == 0 || approximate_message_size_bytes_ < max_buffer_size_bytes_) { stats_.logs_written_.inc(); @@ -315,7 +294,6 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger( + // TODO(shikugawa): configure approximate_critical_message_size_bytes + approximate_critical_message_size_bytes_(max_buffer_size_bytes), log_name_(config.log_name()), + local_info_(local_info) { + critical_client_ = std::make_unique>( client, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs"), @@ -50,6 +53,7 @@ void GrpcAccessLoggerImpl::addCriticalMessageEntry( envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { critical_message_.mutable_message()->mutable_http_logs()->mutable_log_entry()->Add( std::move(entry)); + std::cout << "add" << std::endl; } void GrpcAccessLoggerImpl::addCriticalMessageEntry( @@ -67,6 +71,28 @@ bool GrpcAccessLoggerImpl::isCriticalMessageEmpty() { !critical_message_.message().has_tcp_logs(); } +void GrpcAccessLoggerImpl::flushCriticalMessage() { + if (critical_client_ == nullptr || isCriticalMessageEmpty()) { + return; + } + if (!critical_client_->isStreamStarted()) { + initCriticalMessage(); + } + + approximate_critical_message_size_bytes_ = 0; + critical_client_->flush(critical_message_); + clearCriticalMessage(); +} + +void GrpcAccessLoggerImpl::logCritical(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { + approximate_critical_message_size_bytes_ += entry.ByteSizeLong(); + addCriticalMessageEntry(std::move(entry)); + + if (approximate_critical_message_size_bytes_ <= max_critical_buffer_size_bytes_) { + flushCriticalMessage(); + } +} + void GrpcAccessLoggerImpl::initMessage() { auto* identifier = message_.mutable_identifier(); *identifier->mutable_node() = local_info_.node(); diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 8b337a160fc1e..a1f7e67e759b6 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -31,11 +31,10 @@ struct CriticalAccessLoggerGrpcClientStats { CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) }; +template class CriticalAccessLoggerGrpcClientImpl - : public Common::CriticalAccessLoggerGrpcClient< - envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage> { + : public Common::CriticalAccessLoggerGrpcClient { public: - using RequestType = envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage; using ResponseType = envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse; CriticalAccessLoggerGrpcClientImpl(const Grpc::RawAsyncClientSharedPtr& client, @@ -109,7 +108,7 @@ class CriticalAccessLoggerGrpcClientImpl } } - CriticalAccessLoggerGrpcClientImpl& parent_; + CriticalAccessLoggerGrpcClientImpl& parent_; Grpc::AsyncStream stream_; }; @@ -185,12 +184,11 @@ class CriticalAccessLoggerGrpcClientImpl const uint64_t max_critical_buffer_size_bytes_; }; -class GrpcAccessLoggerImpl : public Common::GrpcAccessLogger< - envoy::data::accesslog::v3::HTTPAccessLogEntry, - envoy::data::accesslog::v3::TCPAccessLogEntry, - envoy::service::accesslog::v3::StreamAccessLogsMessage, - envoy::service::accesslog::v3::StreamAccessLogsResponse, - envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage> { +class GrpcAccessLoggerImpl + : public Common::GrpcAccessLogger { public: GrpcAccessLoggerImpl( const Grpc::RawAsyncClientSharedPtr& client, @@ -200,16 +198,26 @@ class GrpcAccessLoggerImpl : public Common::GrpcAccessLogger< envoy::config::core::v3::ApiVersion transport_api_version); private: + bool isCriticalMessageEmpty(); + void initCriticalMessage(); + void addCriticalMessageEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry); + void addCriticalMessageEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry); + void clearCriticalMessage() { critical_message_.Clear(); } + // Extensions::AccessLoggers::GrpcCommon::GrpcAccessLogger void addEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) override; void addEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) override; - void addCriticalMessageEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) override; - void addCriticalMessageEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) override; bool isEmpty() override; - bool isCriticalMessageEmpty() override; void initMessage() override; - void initCriticalMessage() override; - + void flushCriticalMessage() override; + void logCritical(envoy::data::accesslog::v3::HTTPAccessLogEntry&&) override; + + uint64_t approximate_critical_message_size_bytes_ = 0; + uint64_t max_critical_buffer_size_bytes_; + Common::CriticalAccessLoggerGrpcClientPtr< + envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage> + critical_client_; + envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage critical_message_; const std::string log_name_; const LocalInfo::LocalInfo& local_info_; }; diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc index 9c68e7df909a6..cc84242c93d67 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc @@ -65,11 +65,6 @@ void GrpcAccessLoggerImpl::addEntry(opentelemetry::proto::logs::v1::LogRecord&& bool GrpcAccessLoggerImpl::isEmpty() { return root_->logs().empty(); } -bool GrpcAccessLoggerImpl::isCriticalMessageEmpty() { - // Critical message must be empty in OpenTelemetry logger. - return true; -} - // The message is already initialized in the c'tor, and only the logs are cleared. void GrpcAccessLoggerImpl::initMessage() {} diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h index b83d6d7996dd5..52e61e134ee3b 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h @@ -33,8 +33,7 @@ class GrpcAccessLoggerImpl // as an empty placeholder for the non-used addEntry method. // TODO(itamarkam): Don't cache OpenTelemetry loggers by type (HTTP/TCP). ProtobufWkt::Empty, opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest, - opentelemetry::proto::collector::logs::v1::ExportLogsServiceResponse, - ProtobufWkt::Empty> { + opentelemetry::proto::collector::logs::v1::ExportLogsServiceResponse> { public: GrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, std::string log_name, std::chrono::milliseconds buffer_flush_interval_msec, @@ -48,14 +47,9 @@ class GrpcAccessLoggerImpl void addEntry(opentelemetry::proto::logs::v1::LogRecord&& entry) override; // Non used addEntry method (the above is used for both TCP and HTTP). void addEntry(ProtobufWkt::Empty&& entry) override { (void)entry; }; - void addCriticalMessageEntry(opentelemetry::proto::logs::v1::LogRecord&&) override {} - void addCriticalMessageEntry(ProtobufWkt::Empty&&) override{}; bool isEmpty() override; - bool isCriticalMessageEmpty() override; void initMessage() override; void clearMessage() override; - void clearCriticalMessage() override {} - void initCriticalMessage() override {} opentelemetry::proto::logs::v1::InstrumentationLibraryLogs* root_; }; From e506fd66cf7f32d866105a0ab4128b43006d64d4 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Thu, 5 Aug 2021 15:13:53 +0000 Subject: [PATCH 12/39] fix build Signed-off-by: Shikugawa --- .../common/grpc_access_logger_test.cc | 31 +------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/test/extensions/access_loggers/common/grpc_access_logger_test.cc b/test/extensions/access_loggers/common/grpc_access_logger_test.cc index 6063db35ccb35..ba8891cd4e112 100644 --- a/test/extensions/access_loggers/common/grpc_access_logger_test.cc +++ b/test/extensions/access_loggers/common/grpc_access_logger_test.cc @@ -48,7 +48,7 @@ const Protobuf::MethodDescriptor& mockMethodDescriptor() { // standard Struct and Empty protos. class MockGrpcAccessLoggerImpl : public Common::GrpcAccessLogger { + ProtobufWkt::Struct> { public: MockGrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, std::chrono::milliseconds buffer_flush_interval_msec, @@ -75,16 +75,6 @@ class MockGrpcAccessLoggerImpl 1); } - void mockaddCriticalMessageEntry(const std::string& key) { - if (!critical_message_.fields().contains(key)) { - ProtobufWkt::Value default_value; - default_value.set_number_value(0); - critical_message_.mutable_fields()->insert({key, default_value}); - } - critical_message_.mutable_fields()->at(key).set_number_value( - critical_message_.fields().at(key).number_value() + 1); - } - // Extensions::AccessLoggers::GrpcCommon::GrpcAccessLogger // For testing purposes, we don't really care how each of these virtual methods is implemented, as // it's up to each logger implementation. We test whether they were called in the regular flow of @@ -100,19 +90,7 @@ class MockGrpcAccessLoggerImpl mockAddEntry(MOCK_TCP_LOG_FIELD_NAME); } - void addCriticalMessageEntry(ProtobufWkt::Struct&& entry) override { - (void)entry; - mockaddCriticalMessageEntry(MOCK_HTTP_LOG_FIELD_NAME); - } - - void addCriticalMessageEntry(ProtobufWkt::Empty&& entry) override { - (void)entry; - mockaddCriticalMessageEntry(MOCK_TCP_LOG_FIELD_NAME); - } - bool isEmpty() override { return message_.fields().empty(); } - bool isCriticalMessageEmpty() override { return critical_message_.fields().empty(); } - void initCriticalMessage() override { ++num_critical_inits_; } void initMessage() override { ++num_inits_; } void clearMessage() override { @@ -120,15 +98,8 @@ class MockGrpcAccessLoggerImpl num_clears_++; } - void clearCriticalMessage() override { - critical_message_.Clear(); - num_critical_clears_++; - } - int num_inits_ = 0; - int num_critical_inits_ = 0; int num_clears_ = 0; - int num_critical_clears_ = 0; }; class GrpcAccessLogTest : public testing::Test { From d94bcee0728e52446c123d65bedd17ad9a51d423 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Fri, 6 Aug 2021 07:04:48 +0000 Subject: [PATCH 13/39] fix Signed-off-by: Shikugawa --- source/extensions/access_loggers/grpc/grpc_access_log_impl.cc | 3 +-- source/extensions/access_loggers/grpc/grpc_access_log_impl.h | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 4ce80bb5cbfca..721be7d88f7b0 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -53,7 +53,6 @@ void GrpcAccessLoggerImpl::addCriticalMessageEntry( envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { critical_message_.mutable_message()->mutable_http_logs()->mutable_log_entry()->Add( std::move(entry)); - std::cout << "add" << std::endl; } void GrpcAccessLoggerImpl::addCriticalMessageEntry( @@ -88,7 +87,7 @@ void GrpcAccessLoggerImpl::logCritical(envoy::data::accesslog::v3::HTTPAccessLog approximate_critical_message_size_bytes_ += entry.ByteSizeLong(); addCriticalMessageEntry(std::move(entry)); - if (approximate_critical_message_size_bytes_ <= max_critical_buffer_size_bytes_) { + if (approximate_critical_message_size_bytes_ >= max_critical_buffer_size_bytes_) { flushCriticalMessage(); } } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index a1f7e67e759b6..2872d4e6fa8f6 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -213,7 +213,7 @@ class GrpcAccessLoggerImpl void logCritical(envoy::data::accesslog::v3::HTTPAccessLogEntry&&) override; uint64_t approximate_critical_message_size_bytes_ = 0; - uint64_t max_critical_buffer_size_bytes_; + uint64_t max_critical_buffer_size_bytes_ = 0; Common::CriticalAccessLoggerGrpcClientPtr< envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage> critical_client_; From 75eb0bd889e43f0e3419b340665820e719cf13c9 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 16 Aug 2021 06:07:06 +0000 Subject: [PATCH 14/39] add docs Signed-off-by: Shikugawa --- .../access_loggers/grpc/v3/als.proto | 11 +------ .../access_loggers/grpc/v4alpha/als.proto | 11 +------ api/envoy/service/accesslog/v3/als.proto | 31 +++++++++++++++++-- api/envoy/service/accesslog/v4alpha/als.proto | 31 +++++++++++++++++-- docs/root/version_history/current.rst | 1 + .../access_loggers/grpc/v3/als.proto | 11 +------ .../access_loggers/grpc/v4alpha/als.proto | 11 +------ .../envoy/service/accesslog/v3/als.proto | 31 +++++++++++++++++-- .../envoy/service/accesslog/v4alpha/als.proto | 31 +++++++++++++++++-- tools/spelling/spelling_dictionary.txt | 1 + 10 files changed, 118 insertions(+), 52 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index 5979df747ceec..04c1f8ab772f7 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -90,16 +90,7 @@ message CommonGrpcAccessLogConfig { // Define the log condition as a filter to ensure that the destination is reached. // Logs that match the filter are not forwarded to the normal endpoint, - // but are sent to a special endpoint. This endpoint will return ACK/NACK as a response, - // and Envoy will perform buffering upon receiving a NACK. The buffered message is - // resend until it is reached. Failure to send here means that - // - // 1. when a NACK message corresponding to the message sent is received - // 2. when the ACK message corresponding to the sent message buffer could not be - // received until the message_ack_timeout is reached. - // - // It is recommended to set a strict filter because setting a loose filter here may cause - // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. + // but are sent to a special endpoint. For more detail, config.accesslog.v3.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, diff --git a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 94bcc337286cd..f66d678a8df4a 100644 --- a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -90,16 +90,7 @@ message CommonGrpcAccessLogConfig { // Define the log condition as a filter to ensure that the destination is reached. // Logs that match the filter are not forwarded to the normal endpoint, - // but are sent to a special endpoint. This endpoint will return ACK/NACK as a response, - // and Envoy will perform buffering upon receiving a NACK. The buffered message is - // resend until it is reached. Failure to send here means that - // - // 1. when a NACK message corresponding to the message sent is received - // 2. when the ACK message corresponding to the sent message buffer could not be - // received until the message_ack_timeout is reached. - // - // It is recommended to set a strict filter because setting a loose filter here may cause - // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. + // but are sent to a special endpoint. For more detail, config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, diff --git a/api/envoy/service/accesslog/v3/als.proto b/api/envoy/service/accesslog/v3/als.proto index a07e83be0179a..3a50a4da2cb64 100644 --- a/api/envoy/service/accesslog/v3/als.proto +++ b/api/envoy/service/accesslog/v3/als.proto @@ -21,13 +21,38 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; service AccessLogService { // Envoy will connect and send StreamAccessLogsMessage messages forever. It does not expect any // response to be sent as nothing would be done in the case of failure. The server should - // disconnect if it expects Envoy to reconnect. In the future we may decide to add a different - // API for "critical" access logs in which Envoy will buffer access logs for some period of time - // until it gets an ACK so it could then retry. This API is designed for high throughput with the + // disconnect if it expects Envoy to reconnect. This API is designed for high throughput with the // expectation that it might be lossy. rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } + // This endpoint will be the one that guarantees the arrival of logs with the state set by the user. + // The state of the log can be set in + // :ref:`buffer_log_filter `. + // Log messages that match this filter will be guaranteed to be reachable. In order to guarantee + // the arrival, this endpoint performs the following process. + // + // 1. A response message is returned for each log. The response message includes ACK/NACK status, + // and in case of NACK, the target log is not flushed but buffered by Envoy. In case of NACK, + // it will be buffered by Envoy without flushing the target log. + // 2. Timeout for response message is set and if no message is returned within a certain time, + // it will be considered as unreachable and buffered by Envoy without flushing the target log. This timeout is set by + // :ref:`message_ack_timeout `. + // + // On the ALS receiver side, ACK is expected to be returned to indicate that the log was saved properly, + // and NACK is expected to be returned when the log could not be saved due to some error. + // + // .. attention:: + // + // Buffers for guaranteed reachability can be extremely memory-intensive. Therefore, the following points + // should be considered when using this endpoint. + // + // 1. :ref:`buffer_log_filter ` + // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. + // 2. :ref:`pending_critical_buffer_size_bytes ` + // should be set appropriately to prevent OOM. + // 3. Make sure that ALS receiver is implemented properly. If it is not implemented, all messages will + // be buffered, which may cause OOM soon. rpc BufferedCriticalAccessLogs(stream BufferedCriticalAccessLogsMessage) returns (stream BufferedCriticalAccessLogsResponse) { } diff --git a/api/envoy/service/accesslog/v4alpha/als.proto b/api/envoy/service/accesslog/v4alpha/als.proto index 32d09f6b6bfcf..0c1a685fffa2f 100644 --- a/api/envoy/service/accesslog/v4alpha/als.proto +++ b/api/envoy/service/accesslog/v4alpha/als.proto @@ -21,13 +21,38 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO service AccessLogService { // Envoy will connect and send StreamAccessLogsMessage messages forever. It does not expect any // response to be sent as nothing would be done in the case of failure. The server should - // disconnect if it expects Envoy to reconnect. In the future we may decide to add a different - // API for "critical" access logs in which Envoy will buffer access logs for some period of time - // until it gets an ACK so it could then retry. This API is designed for high throughput with the + // disconnect if it expects Envoy to reconnect. This API is designed for high throughput with the // expectation that it might be lossy. rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } + // This endpoint will be the one that guarantees the arrival of logs with the state set by the user. + // The state of the log can be set in + // :ref:`buffer_log_filter `. + // Log messages that match this filter will be guaranteed to be reachable. In order to guarantee + // the arrival, this endpoint performs the following process. + // + // 1. A response message is returned for each log. The response message includes ACK/NACK status, + // and in case of NACK, the target log is not flushed but buffered by Envoy. In case of NACK, + // it will be buffered by Envoy without flushing the target log. + // 2. Timeout for response message is set and if no message is returned within a certain time, + // it will be considered as unreachable and buffered by Envoy without flushing the target log. This timeout is set by + // :ref:`message_ack_timeout `. + // + // On the ALS receiver side, ACK is expected to be returned to indicate that the log was saved properly, + // and NACK is expected to be returned when the log could not be saved due to some error. + // + // .. attention:: + // + // Buffers for guaranteed reachability can be extremely memory-intensive. Therefore, the following points + // should be considered when using this endpoint. + // + // 1. :ref:`buffer_log_filter ` + // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. + // 2. :ref:`pending_critical_buffer_size_bytes ` + // should be set appropriately to prevent OOM. + // 3. Make sure that ALS receiver is implemented properly. If it is not implemented, all messages will + // be buffered, which may cause OOM soon. rpc BufferedCriticalAccessLogs(stream BufferedCriticalAccessLogsMessage) returns (stream BufferedCriticalAccessLogsResponse) { } diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 9877d650b1233..9668836268297 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -51,6 +51,7 @@ Removed Config or Runtime New Features ------------ +* access log: added critical endpoint to AccessLogService to guarantee log arrival. * http: added :ref:`string_match ` in the header matcher. * http: added support for :ref:`max_requests_per_connection ` for both upstream and downstream connections. diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index 5979df747ceec..04c1f8ab772f7 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -90,16 +90,7 @@ message CommonGrpcAccessLogConfig { // Define the log condition as a filter to ensure that the destination is reached. // Logs that match the filter are not forwarded to the normal endpoint, - // but are sent to a special endpoint. This endpoint will return ACK/NACK as a response, - // and Envoy will perform buffering upon receiving a NACK. The buffered message is - // resend until it is reached. Failure to send here means that - // - // 1. when a NACK message corresponding to the message sent is received - // 2. when the ACK message corresponding to the sent message buffer could not be - // received until the message_ack_timeout is reached. - // - // It is recommended to set a strict filter because setting a loose filter here may cause - // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. + // but are sent to a special endpoint. For more detail, config.accesslog.v3.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index 94bcc337286cd..f66d678a8df4a 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -90,16 +90,7 @@ message CommonGrpcAccessLogConfig { // Define the log condition as a filter to ensure that the destination is reached. // Logs that match the filter are not forwarded to the normal endpoint, - // but are sent to a special endpoint. This endpoint will return ACK/NACK as a response, - // and Envoy will perform buffering upon receiving a NACK. The buffered message is - // resend until it is reached. Failure to send here means that - // - // 1. when a NACK message corresponding to the message sent is received - // 2. when the ACK message corresponding to the sent message buffer could not be - // received until the message_ack_timeout is reached. - // - // It is recommended to set a strict filter because setting a loose filter here may cause - // extreme pressure on the Envoy buffer in cases where the log aggregation cluster is repeatedly restarted. + // but are sent to a special endpoint. For more detail, config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto index a07e83be0179a..3a50a4da2cb64 100644 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v3/als.proto @@ -21,13 +21,38 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; service AccessLogService { // Envoy will connect and send StreamAccessLogsMessage messages forever. It does not expect any // response to be sent as nothing would be done in the case of failure. The server should - // disconnect if it expects Envoy to reconnect. In the future we may decide to add a different - // API for "critical" access logs in which Envoy will buffer access logs for some period of time - // until it gets an ACK so it could then retry. This API is designed for high throughput with the + // disconnect if it expects Envoy to reconnect. This API is designed for high throughput with the // expectation that it might be lossy. rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } + // This endpoint will be the one that guarantees the arrival of logs with the state set by the user. + // The state of the log can be set in + // :ref:`buffer_log_filter `. + // Log messages that match this filter will be guaranteed to be reachable. In order to guarantee + // the arrival, this endpoint performs the following process. + // + // 1. A response message is returned for each log. The response message includes ACK/NACK status, + // and in case of NACK, the target log is not flushed but buffered by Envoy. In case of NACK, + // it will be buffered by Envoy without flushing the target log. + // 2. Timeout for response message is set and if no message is returned within a certain time, + // it will be considered as unreachable and buffered by Envoy without flushing the target log. This timeout is set by + // :ref:`message_ack_timeout `. + // + // On the ALS receiver side, ACK is expected to be returned to indicate that the log was saved properly, + // and NACK is expected to be returned when the log could not be saved due to some error. + // + // .. attention:: + // + // Buffers for guaranteed reachability can be extremely memory-intensive. Therefore, the following points + // should be considered when using this endpoint. + // + // 1. :ref:`buffer_log_filter ` + // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. + // 2. :ref:`pending_critical_buffer_size_bytes ` + // should be set appropriately to prevent OOM. + // 3. Make sure that ALS receiver is implemented properly. If it is not implemented, all messages will + // be buffered, which may cause OOM soon. rpc BufferedCriticalAccessLogs(stream BufferedCriticalAccessLogsMessage) returns (stream BufferedCriticalAccessLogsResponse) { } diff --git a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto index 32d09f6b6bfcf..0c1a685fffa2f 100644 --- a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto @@ -21,13 +21,38 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO service AccessLogService { // Envoy will connect and send StreamAccessLogsMessage messages forever. It does not expect any // response to be sent as nothing would be done in the case of failure. The server should - // disconnect if it expects Envoy to reconnect. In the future we may decide to add a different - // API for "critical" access logs in which Envoy will buffer access logs for some period of time - // until it gets an ACK so it could then retry. This API is designed for high throughput with the + // disconnect if it expects Envoy to reconnect. This API is designed for high throughput with the // expectation that it might be lossy. rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } + // This endpoint will be the one that guarantees the arrival of logs with the state set by the user. + // The state of the log can be set in + // :ref:`buffer_log_filter `. + // Log messages that match this filter will be guaranteed to be reachable. In order to guarantee + // the arrival, this endpoint performs the following process. + // + // 1. A response message is returned for each log. The response message includes ACK/NACK status, + // and in case of NACK, the target log is not flushed but buffered by Envoy. In case of NACK, + // it will be buffered by Envoy without flushing the target log. + // 2. Timeout for response message is set and if no message is returned within a certain time, + // it will be considered as unreachable and buffered by Envoy without flushing the target log. This timeout is set by + // :ref:`message_ack_timeout `. + // + // On the ALS receiver side, ACK is expected to be returned to indicate that the log was saved properly, + // and NACK is expected to be returned when the log could not be saved due to some error. + // + // .. attention:: + // + // Buffers for guaranteed reachability can be extremely memory-intensive. Therefore, the following points + // should be considered when using this endpoint. + // + // 1. :ref:`buffer_log_filter ` + // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. + // 2. :ref:`pending_critical_buffer_size_bytes ` + // should be set appropriately to prevent OOM. + // 3. Make sure that ALS receiver is implemented properly. If it is not implemented, all messages will + // be buffered, which may cause OOM soon. rpc BufferedCriticalAccessLogs(stream BufferedCriticalAccessLogsMessage) returns (stream BufferedCriticalAccessLogsResponse) { } diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index cf48779ec8bac..6f4cb26a3cf68 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -977,6 +977,7 @@ ratelimited ratelimiter rawseti rc +reachability readded readonly readv From 824aa2a4c8a7eb4022f77f4eb064f51b23b04ff6 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 23 Aug 2021 03:01:05 +0000 Subject: [PATCH 15/39] docs Signed-off-by: Shikugawa --- api/envoy/extensions/access_loggers/grpc/v3/als.proto | 5 +++-- .../extensions/access_loggers/grpc/v4alpha/als.proto | 5 +++-- api/envoy/service/accesslog/v3/als.proto | 8 ++++---- api/envoy/service/accesslog/v4alpha/als.proto | 8 ++++---- .../envoy/extensions/access_loggers/grpc/v3/als.proto | 5 +++-- .../extensions/access_loggers/grpc/v4alpha/als.proto | 5 +++-- generated_api_shadow/envoy/service/accesslog/v3/als.proto | 8 ++++---- .../envoy/service/accesslog/v4alpha/als.proto | 8 ++++---- 8 files changed, 28 insertions(+), 24 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index 04c1f8ab772f7..6a362f4ea458f 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -90,15 +90,16 @@ message CommonGrpcAccessLogConfig { // Define the log condition as a filter to ensure that the destination is reached. // Logs that match the filter are not forwarded to the normal endpoint, - // but are sent to a special endpoint. For more detail, + // but are sent to a special endpoint. config.accesslog.v3.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. + // The re-buffered message will be sent again at the next time the critical log is sent. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; - // Soft size limit (in bytes) of the buffer used to store messages during processing in a + // Size limit (in bytes) of the buffer used to store messages during processing in a // critical logger. A critical logger buffers messages until it receives an ACK from upstream. // The default is 16384. google.protobuf.UInt32Value pending_critical_buffer_size_bytes = 9; diff --git a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index f66d678a8df4a..a4bf7d3ea50f7 100644 --- a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -90,15 +90,16 @@ message CommonGrpcAccessLogConfig { // Define the log condition as a filter to ensure that the destination is reached. // Logs that match the filter are not forwarded to the normal endpoint, - // but are sent to a special endpoint. For more detail, + // but are sent to a special endpoint. config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. + // The re-buffered message will be sent again at the next time the critical log is sent. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; - // Soft size limit (in bytes) of the buffer used to store messages during processing in a + // Size limit (in bytes) of the buffer used to store messages during processing in a // critical logger. A critical logger buffers messages until it receives an ACK from upstream. // The default is 16384. google.protobuf.UInt32Value pending_critical_buffer_size_bytes = 9; diff --git a/api/envoy/service/accesslog/v3/als.proto b/api/envoy/service/accesslog/v3/als.proto index 3a50a4da2cb64..74aebf96b81d9 100644 --- a/api/envoy/service/accesslog/v3/als.proto +++ b/api/envoy/service/accesslog/v3/als.proto @@ -26,10 +26,10 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - // This endpoint will be the one that guarantees the arrival of logs with the state set by the user. - // The state of the log can be set in + // This endpoint provides acknowledgment of logs marked as requiring this. + // The requirement for an acknowledgment can be set in // :ref:`buffer_log_filter `. - // Log messages that match this filter will be guaranteed to be reachable. In order to guarantee + // Log messages that match this filter will be guaranteed delivery. In order to guarantee // the arrival, this endpoint performs the following process. // // 1. A response message is returned for each log. The response message includes ACK/NACK status, @@ -78,7 +78,7 @@ message BufferedCriticalAccessLogsResponse { Status status = 1; // Message ID that identifies a message. - uint32 id = 2; + uint64 id = 2; } // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream diff --git a/api/envoy/service/accesslog/v4alpha/als.proto b/api/envoy/service/accesslog/v4alpha/als.proto index 0c1a685fffa2f..c046287427f54 100644 --- a/api/envoy/service/accesslog/v4alpha/als.proto +++ b/api/envoy/service/accesslog/v4alpha/als.proto @@ -26,10 +26,10 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - // This endpoint will be the one that guarantees the arrival of logs with the state set by the user. - // The state of the log can be set in + // This endpoint provides acknowledgment of logs marked as requiring this. + // The requirement for an acknowledgment can be set in // :ref:`buffer_log_filter `. - // Log messages that match this filter will be guaranteed to be reachable. In order to guarantee + // Log messages that match this filter will be guaranteed delivery. In order to guarantee // the arrival, this endpoint performs the following process. // // 1. A response message is returned for each log. The response message includes ACK/NACK status, @@ -81,7 +81,7 @@ message BufferedCriticalAccessLogsResponse { Status status = 1; // Message ID that identifies a message. - uint32 id = 2; + uint64 id = 2; } // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index 04c1f8ab772f7..6a362f4ea458f 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -90,15 +90,16 @@ message CommonGrpcAccessLogConfig { // Define the log condition as a filter to ensure that the destination is reached. // Logs that match the filter are not forwarded to the normal endpoint, - // but are sent to a special endpoint. For more detail, + // but are sent to a special endpoint. config.accesslog.v3.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. + // The re-buffered message will be sent again at the next time the critical log is sent. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; - // Soft size limit (in bytes) of the buffer used to store messages during processing in a + // Size limit (in bytes) of the buffer used to store messages during processing in a // critical logger. A critical logger buffers messages until it receives an ACK from upstream. // The default is 16384. google.protobuf.UInt32Value pending_critical_buffer_size_bytes = 9; diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index f66d678a8df4a..a4bf7d3ea50f7 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -90,15 +90,16 @@ message CommonGrpcAccessLogConfig { // Define the log condition as a filter to ensure that the destination is reached. // Logs that match the filter are not forwarded to the normal endpoint, - // but are sent to a special endpoint. For more detail, + // but are sent to a special endpoint. config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. + // The re-buffered message will be sent again at the next time the critical log is sent. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; - // Soft size limit (in bytes) of the buffer used to store messages during processing in a + // Size limit (in bytes) of the buffer used to store messages during processing in a // critical logger. A critical logger buffers messages until it receives an ACK from upstream. // The default is 16384. google.protobuf.UInt32Value pending_critical_buffer_size_bytes = 9; diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto index 3a50a4da2cb64..74aebf96b81d9 100644 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v3/als.proto @@ -26,10 +26,10 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - // This endpoint will be the one that guarantees the arrival of logs with the state set by the user. - // The state of the log can be set in + // This endpoint provides acknowledgment of logs marked as requiring this. + // The requirement for an acknowledgment can be set in // :ref:`buffer_log_filter `. - // Log messages that match this filter will be guaranteed to be reachable. In order to guarantee + // Log messages that match this filter will be guaranteed delivery. In order to guarantee // the arrival, this endpoint performs the following process. // // 1. A response message is returned for each log. The response message includes ACK/NACK status, @@ -78,7 +78,7 @@ message BufferedCriticalAccessLogsResponse { Status status = 1; // Message ID that identifies a message. - uint32 id = 2; + uint64 id = 2; } // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream diff --git a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto index 0c1a685fffa2f..c046287427f54 100644 --- a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto @@ -26,10 +26,10 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - // This endpoint will be the one that guarantees the arrival of logs with the state set by the user. - // The state of the log can be set in + // This endpoint provides acknowledgment of logs marked as requiring this. + // The requirement for an acknowledgment can be set in // :ref:`buffer_log_filter `. - // Log messages that match this filter will be guaranteed to be reachable. In order to guarantee + // Log messages that match this filter will be guaranteed delivery. In order to guarantee // the arrival, this endpoint performs the following process. // // 1. A response message is returned for each log. The response message includes ACK/NACK status, @@ -81,7 +81,7 @@ message BufferedCriticalAccessLogsResponse { Status status = 1; // Message ID that identifies a message. - uint32 id = 2; + uint64 id = 2; } // Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream From 90225204ed71fb8aec863d9ffddddba6b2522a95 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 25 Aug 2021 10:30:26 +0000 Subject: [PATCH 16/39] fix Signed-off-by: Shikugawa --- api/envoy/extensions/access_loggers/grpc/v3/als.proto | 6 +++--- .../extensions/access_loggers/grpc/v4alpha/als.proto | 6 +++--- api/envoy/service/accesslog/v3/als.proto | 8 ++++---- api/envoy/service/accesslog/v4alpha/als.proto | 8 ++++---- .../envoy/extensions/access_loggers/grpc/v3/als.proto | 6 +++--- .../extensions/access_loggers/grpc/v4alpha/als.proto | 6 +++--- generated_api_shadow/envoy/service/accesslog/v3/als.proto | 8 ++++---- .../envoy/service/accesslog/v4alpha/als.proto | 8 ++++---- .../access_loggers/grpc/http_grpc_access_log_impl.cc | 4 ++-- .../access_loggers/grpc/http_grpc_access_log_impl_test.cc | 2 +- .../grpc/http_grpc_access_log_integration_test.cc | 2 +- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index 6a362f4ea458f..fe20b2f695e5e 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -88,14 +88,14 @@ message CommonGrpcAccessLogConfig { // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; - // Define the log condition as a filter to ensure that the destination is reached. + // Define the log condition for critical access logs. // Logs that match the filter are not forwarded to the normal endpoint, // but are sent to a special endpoint. - config.accesslog.v3.AccessLogFilter buffer_log_filter = 7; + config.accesslog.v3.AccessLogFilter critical_buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time the critical log is sent. + // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter* is queued. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; diff --git a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index a4bf7d3ea50f7..7947c8ff72179 100644 --- a/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -88,14 +88,14 @@ message CommonGrpcAccessLogConfig { // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; - // Define the log condition as a filter to ensure that the destination is reached. + // Define the log condition for critical access logs. // Logs that match the filter are not forwarded to the normal endpoint, // but are sent to a special endpoint. - config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7; + config.accesslog.v4alpha.AccessLogFilter critical_buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time the critical log is sent. + // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter* is queued. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; diff --git a/api/envoy/service/accesslog/v3/als.proto b/api/envoy/service/accesslog/v3/als.proto index 74aebf96b81d9..d053d3b1272f9 100644 --- a/api/envoy/service/accesslog/v3/als.proto +++ b/api/envoy/service/accesslog/v3/als.proto @@ -28,7 +28,7 @@ service AccessLogService { // This endpoint provides acknowledgment of logs marked as requiring this. // The requirement for an acknowledgment can be set in - // :ref:`buffer_log_filter `. + // :ref:`critical_buffer_log_filter `. // Log messages that match this filter will be guaranteed delivery. In order to guarantee // the arrival, this endpoint performs the following process. // @@ -47,7 +47,7 @@ service AccessLogService { // Buffers for guaranteed reachability can be extremely memory-intensive. Therefore, the following points // should be considered when using this endpoint. // - // 1. :ref:`buffer_log_filter ` + // 1. :ref:`critical_buffer_log_filter ` // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. // 2. :ref:`pending_critical_buffer_size_bytes ` // should be set appropriately to prevent OOM. @@ -140,7 +140,7 @@ message BufferedCriticalAccessLogsMessage { // The body of the log message sent to BufferedCriticalAccessLogs. StreamAccessLogsMessage message = 1; - // This is an ID to identify the message, and can be added to the Critical Endpoint - // response message to uniquely identify the message being buffered in the Envoy. + // This is an ID to identify the message, and should be added to the Critical Endpoint + // response message to uniquely identify the message being ACK/NACKed. uint64 id = 4; } diff --git a/api/envoy/service/accesslog/v4alpha/als.proto b/api/envoy/service/accesslog/v4alpha/als.proto index c046287427f54..389177b5dabbf 100644 --- a/api/envoy/service/accesslog/v4alpha/als.proto +++ b/api/envoy/service/accesslog/v4alpha/als.proto @@ -28,7 +28,7 @@ service AccessLogService { // This endpoint provides acknowledgment of logs marked as requiring this. // The requirement for an acknowledgment can be set in - // :ref:`buffer_log_filter `. + // :ref:`critical_buffer_log_filter `. // Log messages that match this filter will be guaranteed delivery. In order to guarantee // the arrival, this endpoint performs the following process. // @@ -47,7 +47,7 @@ service AccessLogService { // Buffers for guaranteed reachability can be extremely memory-intensive. Therefore, the following points // should be considered when using this endpoint. // - // 1. :ref:`buffer_log_filter ` + // 1. :ref:`critical_buffer_log_filter ` // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. // 2. :ref:`pending_critical_buffer_size_bytes ` // should be set appropriately to prevent OOM. @@ -146,7 +146,7 @@ message BufferedCriticalAccessLogsMessage { // The body of the log message sent to BufferedCriticalAccessLogs. StreamAccessLogsMessage message = 1; - // This is an ID to identify the message, and can be added to the Critical Endpoint - // response message to uniquely identify the message being buffered in the Envoy. + // This is an ID to identify the message, and should be added to the Critical Endpoint + // response message to uniquely identify the message being ACK/NACKed. uint64 id = 4; } diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index 6a362f4ea458f..fe20b2f695e5e 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -88,14 +88,14 @@ message CommonGrpcAccessLogConfig { // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; - // Define the log condition as a filter to ensure that the destination is reached. + // Define the log condition for critical access logs. // Logs that match the filter are not forwarded to the normal endpoint, // but are sent to a special endpoint. - config.accesslog.v3.AccessLogFilter buffer_log_filter = 7; + config.accesslog.v3.AccessLogFilter critical_buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time the critical log is sent. + // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter* is queued. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto index a4bf7d3ea50f7..7947c8ff72179 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v4alpha/als.proto @@ -88,14 +88,14 @@ message CommonGrpcAccessLogConfig { // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. repeated string filter_state_objects_to_log = 5; - // Define the log condition as a filter to ensure that the destination is reached. + // Define the log condition for critical access logs. // Logs that match the filter are not forwarded to the normal endpoint, // but are sent to a special endpoint. - config.accesslog.v4alpha.AccessLogFilter buffer_log_filter = 7; + config.accesslog.v4alpha.AccessLogFilter critical_buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time the critical log is sent. + // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter* is queued. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto index 74aebf96b81d9..d053d3b1272f9 100644 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v3/als.proto @@ -28,7 +28,7 @@ service AccessLogService { // This endpoint provides acknowledgment of logs marked as requiring this. // The requirement for an acknowledgment can be set in - // :ref:`buffer_log_filter `. + // :ref:`critical_buffer_log_filter `. // Log messages that match this filter will be guaranteed delivery. In order to guarantee // the arrival, this endpoint performs the following process. // @@ -47,7 +47,7 @@ service AccessLogService { // Buffers for guaranteed reachability can be extremely memory-intensive. Therefore, the following points // should be considered when using this endpoint. // - // 1. :ref:`buffer_log_filter ` + // 1. :ref:`critical_buffer_log_filter ` // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. // 2. :ref:`pending_critical_buffer_size_bytes ` // should be set appropriately to prevent OOM. @@ -140,7 +140,7 @@ message BufferedCriticalAccessLogsMessage { // The body of the log message sent to BufferedCriticalAccessLogs. StreamAccessLogsMessage message = 1; - // This is an ID to identify the message, and can be added to the Critical Endpoint - // response message to uniquely identify the message being buffered in the Envoy. + // This is an ID to identify the message, and should be added to the Critical Endpoint + // response message to uniquely identify the message being ACK/NACKed. uint64 id = 4; } diff --git a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto index c046287427f54..389177b5dabbf 100644 --- a/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v4alpha/als.proto @@ -28,7 +28,7 @@ service AccessLogService { // This endpoint provides acknowledgment of logs marked as requiring this. // The requirement for an acknowledgment can be set in - // :ref:`buffer_log_filter `. + // :ref:`critical_buffer_log_filter `. // Log messages that match this filter will be guaranteed delivery. In order to guarantee // the arrival, this endpoint performs the following process. // @@ -47,7 +47,7 @@ service AccessLogService { // Buffers for guaranteed reachability can be extremely memory-intensive. Therefore, the following points // should be considered when using this endpoint. // - // 1. :ref:`buffer_log_filter ` + // 1. :ref:`critical_buffer_log_filter ` // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. // 2. :ref:`pending_critical_buffer_size_bytes ` // should be set appropriately to prevent OOM. @@ -146,7 +146,7 @@ message BufferedCriticalAccessLogsMessage { // The body of the log message sent to BufferedCriticalAccessLogs. StreamAccessLogsMessage message = 1; - // This is an ID to identify the message, and can be added to the Critical Endpoint - // response message to uniquely identify the message being buffered in the Envoy. + // This is an ID to identify the message, and should be added to the Critical Endpoint + // response message to uniquely identify the message being ACK/NACKed. uint64 id = 4; } diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc index 6b5079e86d7fe..4fa7efb9af111 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc @@ -51,9 +51,9 @@ HttpGrpcAccessLog::HttpGrpcAccessLog( config_.common_config(), transport_version, Common::GrpcAccessLoggerType::HTTP, scope_)); }); - if (config_.has_common_config() && config_.common_config().has_buffer_log_filter()) { + if (config_.has_common_config() && config_.common_config().has_critical_buffer_log_filter()) { critical_log_filter_ = AccessLog::FilterFactory::fromProto( - config_.common_config().buffer_log_filter(), context.runtime(), + config_.common_config().critical_buffer_log_filter(), context.runtime(), context.api().randomGenerator(), context.messageValidationVisitor()); } } diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index 954a1b07537d4..86764714509e9 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -763,7 +763,7 @@ TEST_F(HttpGrpcAccessLogTest, BufferLogFilterTest) { envoy::config::accesslog::v3::AccessLogFilter config; TestUtility::loadFromYaml(filter_yaml, config); - *config_.mutable_common_config()->mutable_buffer_log_filter() = config; + *config_.mutable_common_config()->mutable_critical_buffer_log_filter() = config; init(); diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index 1e7fdfc7269d5..c67f2995b89da 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -158,7 +158,7 @@ class CriticalAccessLogIntegrationTest : public AccessLogIntegrationTest { envoy::config::accesslog::v3::AccessLogFilter filter_config; TestUtility::loadFromYaml(filter_yaml, filter_config); - *common_config->mutable_buffer_log_filter() = filter_config; + *common_config->mutable_critical_buffer_log_filter() = filter_config; access_log->mutable_typed_config()->PackFrom(config); }); From 444dd61683f42fcdef8853530a43ab84f937e354 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Thu, 2 Sep 2021 15:26:55 +0900 Subject: [PATCH 17/39] tmp Signed-off-by: Shikugawa --- api/envoy/service/accesslog/v3/als.proto | 17 +-- .../envoy/service/accesslog/v3/als.proto | 17 +-- .../common/grpc_access_logger.h | 6 - .../grpc/grpc_access_log_impl.cc | 6 +- .../grpc/grpc_access_log_impl.h | 142 ++++++++++++------ .../grpc/grpc_access_log_impl_test.cc | 7 +- .../grpc/http_grpc_access_log_impl_test.cc | 35 ++--- .../http_grpc_access_log_integration_test.cc | 22 +-- 8 files changed, 142 insertions(+), 110 deletions(-) diff --git a/api/envoy/service/accesslog/v3/als.proto b/api/envoy/service/accesslog/v3/als.proto index d053d3b1272f9..8f45e75c49777 100644 --- a/api/envoy/service/accesslog/v3/als.proto +++ b/api/envoy/service/accesslog/v3/als.proto @@ -33,8 +33,7 @@ service AccessLogService { // the arrival, this endpoint performs the following process. // // 1. A response message is returned for each log. The response message includes ACK/NACK status, - // and in case of NACK, the target log is not flushed but buffered by Envoy. In case of NACK, - // it will be buffered by Envoy without flushing the target log. + // and in case of NACK, the target log is not flushed but buffered by Envoy. // 2. Timeout for response message is set and if no message is returned within a certain time, // it will be considered as unreachable and buffered by Envoy without flushing the target log. This timeout is set by // :ref:`message_ack_timeout `. @@ -53,8 +52,8 @@ service AccessLogService { // should be set appropriately to prevent OOM. // 3. Make sure that ALS receiver is implemented properly. If it is not implemented, all messages will // be buffered, which may cause OOM soon. - rpc BufferedCriticalAccessLogs(stream BufferedCriticalAccessLogsMessage) - returns (stream BufferedCriticalAccessLogsResponse) { + rpc CriticalAccessLogs(stream CriticalAccessLogsMessage) + returns (stream CriticalAccessLogsResponse) { } } @@ -64,8 +63,8 @@ message StreamAccessLogsResponse { "envoy.service.accesslog.v2.StreamAccessLogsResponse"; } -// Response received to identify undelivered or delivered messages in BufferedCriticalAccessLogs. -message BufferedCriticalAccessLogsResponse { +// Response received to identify undelivered or delivered messages in CriticalAccessLogs. +message CriticalAccessLogsResponse { enum Status { // Indicates that the message has been received. ACK = 0; @@ -132,12 +131,12 @@ message StreamAccessLogsMessage { } } -// Stream message for the BufferedCriticalAccessLogs API. +// Stream message for the CriticalAccessLogs API. // Envoy opens a stream to the server and streams the access log, // expecting a response. Each message sent is assigned an individual ID, // and the state of the message is tracked based on the ID. -message BufferedCriticalAccessLogsMessage { - // The body of the log message sent to BufferedCriticalAccessLogs. +message CriticalAccessLogsMessage { + // The body of the log message sent to CriticalAccessLogs. StreamAccessLogsMessage message = 1; // This is an ID to identify the message, and should be added to the Critical Endpoint diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto index d053d3b1272f9..8f45e75c49777 100644 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v3/als.proto @@ -33,8 +33,7 @@ service AccessLogService { // the arrival, this endpoint performs the following process. // // 1. A response message is returned for each log. The response message includes ACK/NACK status, - // and in case of NACK, the target log is not flushed but buffered by Envoy. In case of NACK, - // it will be buffered by Envoy without flushing the target log. + // and in case of NACK, the target log is not flushed but buffered by Envoy. // 2. Timeout for response message is set and if no message is returned within a certain time, // it will be considered as unreachable and buffered by Envoy without flushing the target log. This timeout is set by // :ref:`message_ack_timeout `. @@ -53,8 +52,8 @@ service AccessLogService { // should be set appropriately to prevent OOM. // 3. Make sure that ALS receiver is implemented properly. If it is not implemented, all messages will // be buffered, which may cause OOM soon. - rpc BufferedCriticalAccessLogs(stream BufferedCriticalAccessLogsMessage) - returns (stream BufferedCriticalAccessLogsResponse) { + rpc CriticalAccessLogs(stream CriticalAccessLogsMessage) + returns (stream CriticalAccessLogsResponse) { } } @@ -64,8 +63,8 @@ message StreamAccessLogsResponse { "envoy.service.accesslog.v2.StreamAccessLogsResponse"; } -// Response received to identify undelivered or delivered messages in BufferedCriticalAccessLogs. -message BufferedCriticalAccessLogsResponse { +// Response received to identify undelivered or delivered messages in CriticalAccessLogs. +message CriticalAccessLogsResponse { enum Status { // Indicates that the message has been received. ACK = 0; @@ -132,12 +131,12 @@ message StreamAccessLogsMessage { } } -// Stream message for the BufferedCriticalAccessLogs API. +// Stream message for the CriticalAccessLogs API. // Envoy opens a stream to the server and streams the access log, // expecting a response. Each message sent is assigned an individual ID, // and the state of the message is tracked based on the ID. -message BufferedCriticalAccessLogsMessage { - // The body of the log message sent to BufferedCriticalAccessLogs. +message CriticalAccessLogsMessage { + // The body of the log message sent to CriticalAccessLogs. StreamAccessLogsMessage message = 1; // This is an ID to identify the message, and should be added to the Critical Endpoint diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index 386a0a86d48e1..f0248d1e9252d 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -197,12 +197,6 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger>( + critical_client_ = std::make_unique< + CriticalAccessLoggerGrpcClientImpl>( client, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.accesslog.v3.AccessLogService.BufferedCriticalAccessLogs"), + "envoy.service.accesslog.v3.AccessLogService.CriticalAccessLogs"), dispatcher, scope, PROTOBUF_GET_MS_OR_DEFAULT(config, message_ack_timeout, 5000), PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, pending_critical_buffer_size_bytes, 16384)); } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 206c2825ed493..44d174e0f8d2b 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -22,9 +22,11 @@ namespace GrpcCommon { static constexpr absl::string_view GRPC_LOG_STATS_PREFIX = "access_logs.grpc_access_log."; #define CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(COUNTER, GAUGE) \ - COUNTER(critical_logs_succeeded) \ - COUNTER(ciritcal_logs_disposed) \ - COUNTER(pending_timeout) \ + COUNTER(critical_logs_sent) \ + COUNTER(critical_logs_disposed) \ + COUNTER(critical_logs_message_timeout) \ + COUNTER(critical_logs_nack_received) \ + COUNTER(critical_logs_ack_received) \ GAUGE(pending_critical_logs, Accumulate) struct CriticalAccessLoggerGrpcClientStats { @@ -35,33 +37,57 @@ template class CriticalAccessLoggerGrpcClientImpl : public Common::CriticalAccessLoggerGrpcClient { public: - using ResponseType = envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse; + using ResponseType = envoy::service::accesslog::v3::CriticalAccessLogsResponse; + + struct BufferedMessage; + using BufferedMessageRef = std::reference_wrapper; + + // Inflight messages which share same ACK timeout are managed with this timer. + // This avoids to create timers for per inflight messages. + class InflightMessageTimer { + public: + InflightMessageTimer(Event::Dispatcher& dispatcher, + CriticalAccessLoggerGrpcClientStats& stats) { + timer_ = dispatcher.createTimer([this, &stats] { + for (auto&& message_ref : inflight_messages_) { + auto& message = message_ref.get(); + + if (message.state_ == BufferedMessage::State::Pending) { + stats.critical_logs_message_timeout_.inc(); + message.state_ = BufferedMessage::State::Buffered; + } + } + }); + } - CriticalAccessLoggerGrpcClientImpl(const Grpc::RawAsyncClientSharedPtr& client, - const Protobuf::MethodDescriptor& method, - Event::Dispatcher& dispatcher, Stats::Scope& scope, - uint64_t message_ack_timeout, - uint64_t max_critical_buffer_size_bytes) - : client_(client), service_method_(method), dispatcher_(dispatcher), - message_ack_timeout_(message_ack_timeout), - stats_({CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS( - POOL_COUNTER_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()), - POOL_GAUGE_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()))}), - max_critical_buffer_size_bytes_(max_critical_buffer_size_bytes) {} + ~InflightMessageTimer() { timer_->disableTimer(); } + + void add(BufferedMessage& message) { inflight_messages_.push_back(message); } + + void start(std::chrono::milliseconds message_ack_timeout) { + timer_->enableTimer(message_ack_timeout); + } + + private: + std::vector inflight_messages_; + Event::TimerPtr timer_; + }; + + using InflightMessageTimerPtr = std::shared_ptr; struct BufferedMessage { enum class State { - Initial, + Buffered, Pending, }; - Event::TimerPtr timer_; - State state_{State::Initial}; + InflightMessageTimerPtr timer_; + State state_; RequestType message_; }; struct ActiveStream : public Grpc::AsyncStreamCallbacks { - ActiveStream(CriticalAccessLoggerGrpcClientImpl& parent) : parent_(parent) {} + explicit ActiveStream(CriticalAccessLoggerGrpcClientImpl& parent) : parent_(parent) {} // Grpc::AsyncStreamCallbacks void onCreateInitialMetadata(Http::RequestHeaderMap&) override {} @@ -70,21 +96,23 @@ class CriticalAccessLoggerGrpcClientImpl const auto& id = message->id(); if (parent_.buffered_messages_.find(id) != parent_.buffered_messages_.end()) { - // After response wait time exceeded, the state should be Initial. + // After response wait time exceeded, the state should be Buffered. if (parent_.buffered_messages_.at(id).state_ != BufferedMessage::State::Pending) { return; } switch (message->status()) { - case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK: - parent_.stats_.critical_logs_succeeded_.inc(); + case envoy::service::accesslog::v3::CriticalAccessLogsResponse::ACK: + parent_.stats_.critical_logs_ack_received_.inc(); parent_.stats_.pending_critical_logs_.dec(); parent_.current_critical_buffer_size_bytes_ -= parent_.buffered_messages_.at(id).message_.ByteSizeLong(); + ASSERT(parent_.current_critical_buffer_size_bytes_ >= 0); parent_.buffered_messages_.erase(id); break; - case envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK: - parent_.buffered_messages_.at(id).state_ = BufferedMessage::State::Initial; + case envoy::service::accesslog::v3::CriticalAccessLogsResponse::NACK: + parent_.stats_.critical_logs_nack_received_.inc(); + parent_.buffered_messages_.at(id).state_ = BufferedMessage::State::Buffered; break; default: return; @@ -95,7 +123,6 @@ class CriticalAccessLoggerGrpcClientImpl } void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {} void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override { - ASSERT(parent_.active_stream_ != nullptr); if (parent_.active_stream_->stream_ != nullptr) { parent_.active_stream_.reset(); } @@ -105,12 +132,21 @@ class CriticalAccessLoggerGrpcClientImpl Grpc::AsyncStream stream_; }; + CriticalAccessLoggerGrpcClientImpl(const Grpc::RawAsyncClientSharedPtr& client, + const Protobuf::MethodDescriptor& method, + Event::Dispatcher& dispatcher, Stats::Scope& scope, + uint64_t message_ack_timeout, + uint64_t max_critical_buffer_size_bytes) + : client_(client), service_method_(method), dispatcher_(dispatcher), + message_ack_timeout_(message_ack_timeout), + stats_({CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS( + POOL_COUNTER_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()), + POOL_GAUGE_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()))}), + max_critical_buffer_size_bytes_(max_critical_buffer_size_bytes), + active_stream_(std::make_unique(*this)) {} + // Copy messages in the buffer. Take care about memory pressure. void flush(RequestType message) override { - if (!active_stream_) { - active_stream_ = std::make_unique(*this); - } - if (active_stream_->stream_ == nullptr) { active_stream_->stream_ = client_.start(service_method_, *active_stream_, Http::AsyncClient::StreamOptions()); @@ -118,6 +154,7 @@ class CriticalAccessLoggerGrpcClientImpl if (active_stream_->stream_ == nullptr || active_stream_->stream_.isAboveWriteBufferHighWatermark()) { + stats_.critical_logs_disposed_.inc(); active_stream_.reset(); return; } @@ -126,31 +163,29 @@ class CriticalAccessLoggerGrpcClientImpl const auto message_byte_size = message.ByteSizeLong(); if (current_critical_buffer_size_bytes_ + message_byte_size > max_critical_buffer_size_bytes_) { - stats_.ciritcal_logs_disposed_.inc(); + stats_.critical_logs_disposed_.inc(); return; } - buffered_messages_[id] = BufferedMessage{nullptr, BufferedMessage::State::Initial, message}; + buffered_messages_[id] = BufferedMessage{nullptr, BufferedMessage::State::Pending, message}; current_critical_buffer_size_bytes_ += message_byte_size; + ASSERT(current_critical_buffer_size_bytes_ >= 0); stats_.pending_critical_logs_.inc(); + // Creates a timer for each message sent. For messages that failed to be sent in the previous + // transmission, replace the timer used for the previous transmission held by BufferedMessage + // with a　new one to ensure that the old timer is disabled. + InflightMessageTimerPtr inflight_timer = + std::make_shared(dispatcher_, stats_); + for (auto&& buffered_message : buffered_messages_) { - uint32_t id = buffered_message.first; + const uint32_t id = buffered_message.first; buffered_message.second.message_.set_id(id); - buffered_message.second.state_ = BufferedMessage::State::Pending; - - buffered_message.second.timer_ = dispatcher_.createTimer([&]() { - if (buffered_message.second.state_ == BufferedMessage::State::Pending) { - stats_.pending_timeout_.inc(); - buffered_message.second.state_ = BufferedMessage::State::Initial; - buffered_message.second.timer_->disableTimer(); - buffered_message.second.timer_.reset(); - } - }); - buffered_message.second.timer_->enableTimer(message_ack_timeout_); - - active_stream_->stream_->sendMessage(buffered_message.second.message_, false); + inflight_timer->add(buffered_message.second); + buffered_message.second.timer_ = inflight_timer; } + + sendMessageAll(); } bool isStreamStarted() override { @@ -158,10 +193,20 @@ class CriticalAccessLoggerGrpcClientImpl } private: + void sendMessageAll() { + ASSERT(buffered_messages_.size() != 0); + auto& inflight_message_timer = buffered_messages_.begin()->second.timer_; + + for (auto&& buffered_message : buffered_messages_) { + stats_.critical_logs_sent_.inc(); + active_stream_->stream_->sendMessage(buffered_message.second.message_, false); + } + inflight_message_timer->start(message_ack_timeout_); + } + friend ActiveStream; absl::flat_hash_map buffered_messages_; - std::unique_ptr active_stream_; Grpc::AsyncClient client_; const Protobuf::MethodDescriptor& service_method_; Event::Dispatcher& dispatcher_; @@ -169,6 +214,7 @@ class CriticalAccessLoggerGrpcClientImpl CriticalAccessLoggerGrpcClientStats stats_; uint64_t current_critical_buffer_size_bytes_ = 0; const uint64_t max_critical_buffer_size_bytes_; + std::unique_ptr active_stream_; }; class GrpcAccessLoggerImpl @@ -201,9 +247,9 @@ class GrpcAccessLoggerImpl uint64_t approximate_critical_message_size_bytes_ = 0; uint64_t max_critical_buffer_size_bytes_ = 0; Common::CriticalAccessLoggerGrpcClientPtr< - envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage> + envoy::service::accesslog::v3::CriticalAccessLogsMessage> critical_client_; - envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage critical_message_; + envoy::service::accesslog::v3::CriticalAccessLogsMessage critical_message_; const std::string log_name_; const LocalInfo::LocalInfo& local_info_; }; diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 2bfecc730a64e..809cfc5a579eb 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -60,12 +60,12 @@ class GrpcAccessLoggerImplTestHelper { } void expectStreamCriticalMessage(const std::string& expected_message_yaml) { - envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage expected_message; + envoy::service::accesslog::v3::CriticalAccessLogsMessage expected_message; TestUtility::loadFromYaml(expected_message_yaml, expected_message); EXPECT_CALL(stream_, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); EXPECT_CALL(stream_, sendMessageRaw_(_, false)) .WillOnce(Invoke([expected_message](Buffer::InstancePtr& request, bool) { - envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage message; + envoy::service::accesslog::v3::CriticalAccessLogsMessage message; Buffer::ZeroCopyInputStreamImpl request_stream(std::move(request)); EXPECT_TRUE(message.ParseFromZeroCopyStream(&request_stream)); EXPECT_GT(message.id(), 0); @@ -148,6 +148,7 @@ class CriticalGrpcAccessLoggerImplTest : public GrpcAccessLoggerImplTest { CriticalGrpcAccessLoggerImplTest() { mock_buffer_timer_ = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*mock_buffer_timer_, enableTimer(_, _)); + EXPECT_CALL(*mock_buffer_timer_, disableTimer()); } private: @@ -205,7 +206,7 @@ TEST_F(CriticalGrpcAccessLoggerImplBufferLimitTest, BasicBehavior) { entry.mutable_request()->set_path("/test/path1"); logger_->log(envoy::data::accesslog::v3::HTTPAccessLogEntry(entry), true); EXPECT_EQ( - stats_store_.counterFromString("access_logs.grpc_access_log.ciritcal_logs_disposed").value(), + stats_store_.counterFromString("access_logs.grpc_access_log.critical_logs_disposed").value(), 1); } diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index 3e4e7a6d64690..adaf003a10f70 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -75,7 +75,7 @@ class HttpGrpcAccessLogTest : public testing::Test { logger_cache_, factory_context_); } - void expectLog(const std::string& expected_log_entry_yaml, bool expect_critical) { + void expectLog(const std::string& expected_log_entry_yaml, bool expect_critical = false) { if (access_log_ == nullptr) { init(); } @@ -119,8 +119,7 @@ class HttpGrpcAccessLogTest : public testing::Test { request_headers_bytes: {} response: {{}} )EOF", - request_method, request_method.length() + 7), - false); + request_method, request_method.length() + 7)); access_log_->log(&request_headers, nullptr, nullptr, stream_info); } @@ -201,8 +200,7 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { value: 10s request: {} response: {} -)EOF", - false); +)EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -232,8 +230,7 @@ response: {} nanos: 2000000 request: {} response: {} -)EOF", - false); +)EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -335,8 +332,7 @@ protocol_version: HTTP10 response_headers_bytes: 10 response_body_bytes: 20 response_code_details: "via_upstream" -)EOF", - false); +)EOF"); access_log_->log(&request_headers, &response_headers, nullptr, stream_info); } @@ -371,8 +367,7 @@ protocol_version: HTTP10 request_method: "METHOD_UNSPECIFIED" request_headers_bytes: 16 response: {} -)EOF", - false); +)EOF"); access_log_->log(&request_headers, nullptr, nullptr, stream_info); } @@ -438,8 +433,7 @@ response: {} request_method: "METHOD_UNSPECIFIED" request_headers_bytes: 16 response: {} -)EOF", - false); +)EOF"); access_log_->log(&request_headers, nullptr, nullptr, stream_info); } @@ -489,8 +483,7 @@ response: {} request: request_method: "METHOD_UNSPECIFIED" response: {} -)EOF", - false); +)EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -540,8 +533,7 @@ response: {} request: request_method: "METHOD_UNSPECIFIED" response: {} -)EOF", - false); +)EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -591,8 +583,7 @@ response: {} request: request_method: "METHOD_UNSPECIFIED" response: {} -)EOF", - false); +)EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -642,8 +633,7 @@ response: {} request: request_method: "METHOD_UNSPECIFIED" response: {} -)EOF", - false); +)EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } } @@ -730,8 +720,7 @@ TEST_F(HttpGrpcAccessLogTest, MarshallingAdditionalHeaders) { response_trailers: "x-logged-trailer": "value" "x-empty-trailer": "" -)EOF", - false); +)EOF"); access_log_->log(&request_headers, &response_headers, &response_trailers, stream_info); } } diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index 1f2611f7551d4..57fcc18d98683 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -155,14 +155,14 @@ class CriticalAccessLogIntegrationTest : public AccessLogIntegrationTest { ABSL_MUST_USE_RESULT AssertionResult waitForCriticalAccessLogRequest(const std::string& expected_request_msg_yaml) { - envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage request_msg; + envoy::service::accesslog::v3::CriticalAccessLogsMessage request_msg; VERIFY_ASSERTION(access_log_request_->waitForGrpcMessage(*dispatcher_, request_msg)); EXPECT_EQ("POST", access_log_request_->headers().getMethodValue()); - EXPECT_EQ("/envoy.service.accesslog.v3.AccessLogService/BufferedCriticalAccessLogs", + EXPECT_EQ("/envoy.service.accesslog.v3.AccessLogService/CriticalAccessLogs", access_log_request_->headers().getPathValue()); EXPECT_EQ("application/grpc", access_log_request_->headers().getContentTypeValue()); - envoy::service::accesslog::v3::BufferedCriticalAccessLogsMessage expected_request_msg; + envoy::service::accesslog::v3::CriticalAccessLogsMessage expected_request_msg; TestUtility::loadFromYaml(expected_request_msg_yaml, expected_request_msg); // Clear fields which are not deterministic. @@ -229,10 +229,10 @@ TEST_P(CriticalAccessLogIntegrationTest, BasicAckFlow) { )EOF"))); access_log_request_->startGrpcStream(); - envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse response_msg; + envoy::service::accesslog::v3::CriticalAccessLogsResponse response_msg; response_msg.set_id(pending_message_id_); pending_message_id_ = 0; - response_msg.set_status(envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::ACK); + response_msg.set_status(envoy::service::accesslog::v3::CriticalAccessLogsResponse::ACK); access_log_request_->sendGrpcMessage(response_msg); access_log_request_->finishGrpcStream(Grpc::Status::Ok); switch (clientType()) { @@ -246,7 +246,8 @@ TEST_P(CriticalAccessLogIntegrationTest, BasicAckFlow) { NOT_REACHED_GCOVR_EXCL_LINE; } - test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_succeeded", 1); + test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_sent", 1); + test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_ack_received", 1); test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 0); cleanup(); } @@ -286,10 +287,10 @@ TEST_P(CriticalAccessLogIntegrationTest, BasicNackFlow) { )EOF"))); access_log_request_->startGrpcStream(); - envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse response_msg; + envoy::service::accesslog::v3::CriticalAccessLogsResponse response_msg; response_msg.set_id(pending_message_id_); pending_message_id_ = 0; - response_msg.set_status(envoy::service::accesslog::v3::BufferedCriticalAccessLogsResponse::NACK); + response_msg.set_status(envoy::service::accesslog::v3::CriticalAccessLogsResponse::NACK); access_log_request_->sendGrpcMessage(response_msg); access_log_request_->finishGrpcStream(Grpc::Status::Ok); switch (clientType()) { @@ -303,6 +304,8 @@ TEST_P(CriticalAccessLogIntegrationTest, BasicNackFlow) { NOT_REACHED_GCOVR_EXCL_LINE; } + test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_sent", 1); + test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_nack_received", 1); test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 1); cleanup(); } @@ -341,7 +344,8 @@ TEST_P(CriticalAccessLogIntegrationTest, NoResponseFlow) { response_headers_bytes: 54 )EOF"))); - test_server_->waitForCounterEq("access_logs.grpc_access_log.pending_timeout", 1); + test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_sent", 1); + test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_message_timeout", 1); test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 1); cleanup(); } From 6c4cb6091ab07935c2b218293d61da236185fb56 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Thu, 2 Sep 2021 16:46:08 +0900 Subject: [PATCH 18/39] api review Signed-off-by: Shikugawa --- api/envoy/extensions/access_loggers/grpc/v3/als.proto | 2 +- api/envoy/service/accesslog/v3/als.proto | 2 +- docs/root/version_history/current.rst | 2 +- .../envoy/extensions/access_loggers/grpc/v3/als.proto | 2 +- generated_api_shadow/envoy/service/accesslog/v3/als.proto | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index fe20b2f695e5e..9512842d2df8f 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -95,7 +95,7 @@ message CommonGrpcAccessLogConfig { // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter* is queued. + // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter is queued. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; diff --git a/api/envoy/service/accesslog/v3/als.proto b/api/envoy/service/accesslog/v3/als.proto index 8f45e75c49777..918208472841b 100644 --- a/api/envoy/service/accesslog/v3/als.proto +++ b/api/envoy/service/accesslog/v3/als.proto @@ -26,7 +26,7 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - // This endpoint provides acknowledgment of logs marked as requiring this. + // This endpoint provides acknowledgment of logs marked as requiring acknowledgment. // The requirement for an acknowledgment can be set in // :ref:`critical_buffer_log_filter `. // Log messages that match this filter will be guaranteed delivery. In order to guarantee diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index cf6b8d5f26610..cc3de8bb943e6 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -84,7 +84,7 @@ Removed Config or Runtime New Features ------------ -* access log: added critical endpoint to AccessLogService to guarantee log arrival. +* access log: added :ref:`critical endpoint ` to AccessLogService to guarantee log arrival. * access_log: added :ref:`METADATA` token to handle all types of metadata (DYNAMIC, CLUSTER, ROUTE). * bootstrap: added :ref:`inline_headers ` in the bootstrap to make custom inline headers bootstrap configurable. * contrib: added new :ref:`contrib images ` which contain contrib extensions. diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index fe20b2f695e5e..9512842d2df8f 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -95,7 +95,7 @@ message CommonGrpcAccessLogConfig { // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter* is queued. + // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter is queued. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto index 8f45e75c49777..918208472841b 100644 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v3/als.proto @@ -26,7 +26,7 @@ service AccessLogService { rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { } - // This endpoint provides acknowledgment of logs marked as requiring this. + // This endpoint provides acknowledgment of logs marked as requiring acknowledgment. // The requirement for an acknowledgment can be set in // :ref:`critical_buffer_log_filter `. // Log messages that match this filter will be guaranteed delivery. In order to guarantee From ee43245a931067d1388708218acfc17e36a5c961 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Thu, 2 Sep 2021 21:42:32 +0900 Subject: [PATCH 19/39] fix Signed-off-by: Shikugawa --- .../access_loggers/grpc/v3/als.proto | 4 +- api/envoy/service/accesslog/v3/als.proto | 2 +- docs/root/version_history/current.rst | 2 +- .../access_loggers/grpc/v3/als.proto | 4 +- .../envoy/service/accesslog/v3/als.proto | 2 +- .../grpc/grpc_access_log_impl.cc | 6 +-- .../grpc/grpc_access_log_impl.h | 40 +++++++++++-------- .../grpc/grpc_access_log_impl_test.cc | 2 +- 8 files changed, 35 insertions(+), 27 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index 9512842d2df8f..bdf0fea2efc00 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -95,12 +95,12 @@ message CommonGrpcAccessLogConfig { // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter is queued. + // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter* is queued. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; // Size limit (in bytes) of the buffer used to store messages during processing in a // critical logger. A critical logger buffers messages until it receives an ACK from upstream. // The default is 16384. - google.protobuf.UInt32Value pending_critical_buffer_size_bytes = 9; + google.protobuf.UInt32Value max_pending_buffer_size_bytes = 9; } diff --git a/api/envoy/service/accesslog/v3/als.proto b/api/envoy/service/accesslog/v3/als.proto index 918208472841b..4f594130c2d3b 100644 --- a/api/envoy/service/accesslog/v3/als.proto +++ b/api/envoy/service/accesslog/v3/als.proto @@ -48,7 +48,7 @@ service AccessLogService { // // 1. :ref:`critical_buffer_log_filter ` // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. - // 2. :ref:`pending_critical_buffer_size_bytes ` + // 2. :ref:`max_pending_buffer_size_bytes ` // should be set appropriately to prevent OOM. // 3. Make sure that ALS receiver is implemented properly. If it is not implemented, all messages will // be buffered, which may cause OOM soon. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index cc3de8bb943e6..3e94a9894daff 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -84,7 +84,7 @@ Removed Config or Runtime New Features ------------ -* access log: added :ref:`critical endpoint ` to AccessLogService to guarantee log arrival. +* access log: added :ref:`critical log message ` to AccessLogService to guarantee log arrival. * access_log: added :ref:`METADATA` token to handle all types of metadata (DYNAMIC, CLUSTER, ROUTE). * bootstrap: added :ref:`inline_headers ` in the bootstrap to make custom inline headers bootstrap configurable. * contrib: added new :ref:`contrib images ` which contain contrib extensions. diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index 9512842d2df8f..bdf0fea2efc00 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -95,12 +95,12 @@ message CommonGrpcAccessLogConfig { // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter is queued. + // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter* is queued. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; // Size limit (in bytes) of the buffer used to store messages during processing in a // critical logger. A critical logger buffers messages until it receives an ACK from upstream. // The default is 16384. - google.protobuf.UInt32Value pending_critical_buffer_size_bytes = 9; + google.protobuf.UInt32Value max_pending_buffer_size_bytes = 9; } diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto index 918208472841b..4f594130c2d3b 100644 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ b/generated_api_shadow/envoy/service/accesslog/v3/als.proto @@ -48,7 +48,7 @@ service AccessLogService { // // 1. :ref:`critical_buffer_log_filter ` // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. - // 2. :ref:`pending_critical_buffer_size_bytes ` + // 2. :ref:`max_pending_buffer_size_bytes ` // should be set appropriately to prevent OOM. // 3. Make sure that ALS receiver is implemented properly. If it is not implemented, all messages will // be buffered, which may cause OOM soon. diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index bf0e440325f5a..4e132607ce842 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -24,7 +24,7 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( dispatcher, scope, GRPC_LOG_STATS_PREFIX.data(), *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v3.AccessLogService.StreamAccessLogs")), - approximate_critical_message_size_bytes_(max_buffer_size_bytes), log_name_(config.log_name()), + max_critical_message_size_bytes_(max_buffer_size_bytes), log_name_(config.log_name()), local_info_(local_info) { critical_client_ = std::make_unique< CriticalAccessLoggerGrpcClientImpl>( @@ -32,7 +32,7 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v3.AccessLogService.CriticalAccessLogs"), dispatcher, scope, PROTOBUF_GET_MS_OR_DEFAULT(config, message_ack_timeout, 5000), - PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, pending_critical_buffer_size_bytes, 16384)); + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_pending_buffer_size_bytes, 16384)); } void GrpcAccessLoggerImpl::addEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { @@ -81,7 +81,7 @@ void GrpcAccessLoggerImpl::logCritical(envoy::data::accesslog::v3::HTTPAccessLog approximate_critical_message_size_bytes_ += entry.ByteSizeLong(); addCriticalMessageEntry(std::move(entry)); - if (approximate_critical_message_size_bytes_ >= max_critical_buffer_size_bytes_) { + if (approximate_critical_message_size_bytes_ >= max_critical_message_size_bytes_) { flushCriticalMessage(); } } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 44d174e0f8d2b..13304db5cec5b 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -97,7 +97,7 @@ class CriticalAccessLoggerGrpcClientImpl if (parent_.buffered_messages_.find(id) != parent_.buffered_messages_.end()) { // After response wait time exceeded, the state should be Buffered. - if (parent_.buffered_messages_.at(id).state_ != BufferedMessage::State::Pending) { + if (parent_.buffered_messages_.at(id).state_ == BufferedMessage::State::Buffered) { return; } @@ -105,9 +105,9 @@ class CriticalAccessLoggerGrpcClientImpl case envoy::service::accesslog::v3::CriticalAccessLogsResponse::ACK: parent_.stats_.critical_logs_ack_received_.inc(); parent_.stats_.pending_critical_logs_.dec(); - parent_.current_critical_buffer_size_bytes_ -= + parent_.current_pending_buffer_size_bytes_ -= parent_.buffered_messages_.at(id).message_.ByteSizeLong(); - ASSERT(parent_.current_critical_buffer_size_bytes_ >= 0); + ASSERT(parent_.current_pending_buffer_size_bytes_ >= 0); parent_.buffered_messages_.erase(id); break; case envoy::service::accesslog::v3::CriticalAccessLogsResponse::NACK: @@ -123,8 +123,8 @@ class CriticalAccessLoggerGrpcClientImpl } void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {} void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override { - if (parent_.active_stream_->stream_ != nullptr) { - parent_.active_stream_.reset(); + if (stream_ != nullptr) { + stream_ = nullptr; } } @@ -136,13 +136,13 @@ class CriticalAccessLoggerGrpcClientImpl const Protobuf::MethodDescriptor& method, Event::Dispatcher& dispatcher, Stats::Scope& scope, uint64_t message_ack_timeout, - uint64_t max_critical_buffer_size_bytes) + uint64_t max_pending_buffer_size_bytes) : client_(client), service_method_(method), dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout), stats_({CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS( POOL_COUNTER_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()), POOL_GAUGE_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()))}), - max_critical_buffer_size_bytes_(max_critical_buffer_size_bytes), + max_pending_buffer_size_bytes_(max_pending_buffer_size_bytes), active_stream_(std::make_unique(*this)) {} // Copy messages in the buffer. Take care about memory pressure. @@ -152,8 +152,7 @@ class CriticalAccessLoggerGrpcClientImpl client_.start(service_method_, *active_stream_, Http::AsyncClient::StreamOptions()); } - if (active_stream_->stream_ == nullptr || - active_stream_->stream_.isAboveWriteBufferHighWatermark()) { + if (active_stream_->stream_.isAboveWriteBufferHighWatermark()) { stats_.critical_logs_disposed_.inc(); active_stream_.reset(); return; @@ -162,14 +161,14 @@ class CriticalAccessLoggerGrpcClientImpl uint32_t id = MessageUtil::hash(message); const auto message_byte_size = message.ByteSizeLong(); - if (current_critical_buffer_size_bytes_ + message_byte_size > max_critical_buffer_size_bytes_) { + if (current_pending_buffer_size_bytes_ + message_byte_size > max_pending_buffer_size_bytes_) { stats_.critical_logs_disposed_.inc(); return; } - buffered_messages_[id] = BufferedMessage{nullptr, BufferedMessage::State::Pending, message}; - current_critical_buffer_size_bytes_ += message_byte_size; - ASSERT(current_critical_buffer_size_bytes_ >= 0); + buffered_messages_[id] = BufferedMessage{nullptr, BufferedMessage::State::Buffered, message}; + current_pending_buffer_size_bytes_ += message_byte_size; + ASSERT(current_pending_buffer_size_bytes_ >= 0); stats_.pending_critical_logs_.inc(); // Creates a timer for each message sent. For messages that failed to be sent in the previous @@ -179,6 +178,10 @@ class CriticalAccessLoggerGrpcClientImpl std::make_shared(dispatcher_, stats_); for (auto&& buffered_message : buffered_messages_) { + if (buffered_message.second.state_ == BufferedMessage::State::Pending) { + continue; + } + const uint32_t id = buffered_message.first; buffered_message.second.message_.set_id(id); inflight_timer->add(buffered_message.second); @@ -198,6 +201,11 @@ class CriticalAccessLoggerGrpcClientImpl auto& inflight_message_timer = buffered_messages_.begin()->second.timer_; for (auto&& buffered_message : buffered_messages_) { + if (buffered_message.second.state_ == BufferedMessage::State::Pending) { + continue; + } + + buffered_message.second.state_ = BufferedMessage::State::Pending; stats_.critical_logs_sent_.inc(); active_stream_->stream_->sendMessage(buffered_message.second.message_, false); } @@ -212,8 +220,8 @@ class CriticalAccessLoggerGrpcClientImpl Event::Dispatcher& dispatcher_; std::chrono::milliseconds message_ack_timeout_; CriticalAccessLoggerGrpcClientStats stats_; - uint64_t current_critical_buffer_size_bytes_ = 0; - const uint64_t max_critical_buffer_size_bytes_; + uint64_t current_pending_buffer_size_bytes_ = 0; + const uint64_t max_pending_buffer_size_bytes_; std::unique_ptr active_stream_; }; @@ -245,7 +253,7 @@ class GrpcAccessLoggerImpl void logCritical(envoy::data::accesslog::v3::HTTPAccessLogEntry&&) override; uint64_t approximate_critical_message_size_bytes_ = 0; - uint64_t max_critical_buffer_size_bytes_ = 0; + uint64_t max_critical_message_size_bytes_ = 0; Common::CriticalAccessLoggerGrpcClientPtr< envoy::service::accesslog::v3::CriticalAccessLogsMessage> critical_client_; diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 809cfc5a579eb..43bb9ca06a251 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -183,7 +183,7 @@ class CriticalGrpcAccessLoggerImplBufferLimitTest : public testing::Test { grpc_access_logger_impl_test_helper_(local_info_, async_client_) { EXPECT_CALL(*timer_, enableTimer(_, _)); config_.set_log_name("test_log_name"); - config_.mutable_pending_critical_buffer_size_bytes()->set_value(0); + config_.mutable_max_pending_buffer_size_bytes()->set_value(0); logger_ = std::make_unique(Grpc::RawAsyncClientPtr{async_client_}, config_, FlushInterval, BUFFER_SIZE_BYTES, dispatcher_, local_info_, stats_store_); From 9d7e6d9e778faf3ee420e60841ce5d487ff4d478 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Thu, 2 Sep 2021 21:46:46 +0900 Subject: [PATCH 20/39] fix Signed-off-by: Shikugawa --- api/envoy/extensions/access_loggers/grpc/v3/als.proto | 2 +- .../envoy/extensions/access_loggers/grpc/v3/als.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index bdf0fea2efc00..29904142368e1 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -95,7 +95,7 @@ message CommonGrpcAccessLogConfig { // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter* is queued. + // The re-buffered message will be sent again at the next time a log matching *critical_buffer_log_filter* is queued. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto index bdf0fea2efc00..29904142368e1 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -95,7 +95,7 @@ message CommonGrpcAccessLogConfig { // The time to wait for an ACK message. If no ACK message is returned after this time, // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time a log matching *critical_critical_buffer_log_filter* is queued. + // The re-buffered message will be sent again at the next time a log matching *critical_buffer_log_filter* is queued. google.protobuf.Duration message_ack_timeout = 8 [(validate.rules).duration = {gte {nanos: 1000000}}]; From 1a72c4e649a9d0196933ab6bfd4613ac0cbb3c02 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Sat, 4 Sep 2021 09:51:51 +0900 Subject: [PATCH 21/39] tmp Signed-off-by: Shikugawa --- source/common/grpc/BUILD | 9 + .../common/grpc/buffered_async_client_impl.h | 107 +++++++++ .../common/grpc_access_logger.h | 19 -- source/extensions/access_loggers/grpc/BUILD | 1 + .../grpc/grpc_access_log_impl.cc | 35 ++- .../grpc/grpc_access_log_impl.h | 218 +++++------------- test/common/grpc/BUILD | 15 ++ .../grpc/buffered_async_client_impl_test.cc | 86 +++++++ .../grpc/grpc_access_log_impl_test.cc | 34 --- .../http_grpc_access_log_integration_test.cc | 3 - 10 files changed, 304 insertions(+), 223 deletions(-) create mode 100644 source/common/grpc/buffered_async_client_impl.h create mode 100644 test/common/grpc/buffered_async_client_impl_test.cc diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index 8e3cc89fd7e77..1cd01b17ba921 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -206,3 +206,12 @@ envoy_cc_library( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) + +envoy_cc_library( + name = "buffered_async_client_lib", + hdrs = ["buffered_async_client_impl.h"], + deps = [ + ":typed_async_client_lib", + "//source/common/protobuf:utility_lib", + ], +) diff --git a/source/common/grpc/buffered_async_client_impl.h b/source/common/grpc/buffered_async_client_impl.h new file mode 100644 index 0000000000000..620d58fa63bd0 --- /dev/null +++ b/source/common/grpc/buffered_async_client_impl.h @@ -0,0 +1,107 @@ +#pragma once + +#include "source/common/grpc/typed_async_client.h" +#include "source/common/protobuf/utility.h" + +namespace Envoy { +namespace Grpc { + +enum class BufferState { Buffered, Pending }; + +template class BufferedAsyncClient { +public: + BufferedAsyncClient(uint32_t max_buffer_bytes, const Protobuf::MethodDescriptor& service_method, + Grpc::AsyncStreamCallbacks& callbacks, + const Grpc::AsyncClient& client) + : max_buffer_bytes_(max_buffer_bytes), service_method_(service_method), callbacks_(callbacks), + client_(client) {} + + ~BufferedAsyncClient() { + if (active_stream_ != nullptr) { + // active_stream_->resetStream(); + } + } + + uint32_t publishId(RequestType& message) { return MessageUtil::hash(message); } + + void bufferMessage(uint32_t id, RequestType& message) { + const auto buffer_size = message.ByteSizeLong(); + if (current_buffer_bytes_ + buffer_size > max_buffer_bytes_) { + return; + } + + message_buffer_[id] = std::make_pair(BufferState::Buffered, message); + current_buffer_bytes_ += max_buffer_bytes_; + } + + void bufferMessage(uint32_t id) { + if (message_buffer_.find(id) == message_buffer_.end()) { + return; + } + message_buffer_.at(id).first = BufferState::Buffered; + } + + std::vector sendBufferedMessages() { + if (active_stream_ == nullptr) { + active_stream_ = + client_.start(service_method_, callbacks_, Http::AsyncClient::StreamOptions()); + } + std::vector inflight_message_ids; + + if (active_stream_->isAboveWriteBufferHighWatermark()) { + resetStream(); + return inflight_message_ids; + } + for (auto&& it : message_buffer_) { + const auto id = it.first; + auto& state = it.second.first; + auto& message = it.second.second; + + if (state == BufferState::Pending) { + continue; + } + + state = BufferState::Pending; + inflight_message_ids.push_back(id); + active_stream_->sendMessage(message, false); + } + + return inflight_message_ids; + } + + void clearPendingMessage(uint32_t id) { + if (message_buffer_.find(id) == message_buffer_.end()) { + return; + } + auto& buffer = message_buffer_.at(id); + + if (buffer.first == BufferState::Pending) { + const auto buffer_size = buffer.second.ByteSizeLong(); + current_buffer_bytes_ -= buffer_size; + message_buffer_.erase(id); + } + } + + void resetStream() { + if (active_stream_ != nullptr) { + active_stream_ = nullptr; + } + } + + bool hasActiveStream() { return active_stream_ != nullptr; } + +private: + uint32_t max_buffer_bytes_ = 0; + const Protobuf::MethodDescriptor& service_method_; + Grpc::AsyncStreamCallbacks& callbacks_; + Grpc::AsyncClient client_; + Grpc::AsyncStream active_stream_; + absl::flat_hash_map> message_buffer_; + uint32_t current_buffer_bytes_ = 0; +}; + +template +using BufferedAsyncClientPtr = std::unique_ptr>; + +} // namespace Grpc +} // namespace Envoy diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index f0248d1e9252d..ab86a357f3756 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -150,25 +150,6 @@ struct GrpcAccessLoggerStats { ALL_GRPC_ACCESS_LOGGER_STATS(GENERATE_COUNTER_STRUCT) }; -template class CriticalAccessLoggerGrpcClient { -public: - virtual ~CriticalAccessLoggerGrpcClient() = default; - - /** - * Flush critical messages. - */ - virtual void flush(RequestType message) PURE; - - /** - * Whether this client has active stream or not. - */ - virtual bool isStreamStarted() PURE; -}; - -template -using CriticalAccessLoggerGrpcClientPtr = - std::unique_ptr>; - /** * Base class for defining a gRPC logger with the `HttpLogProto` and `TcpLogProto` access log * entries and `LogRequest` and `LogResponse` gRPC messages. diff --git a/source/extensions/access_loggers/grpc/BUILD b/source/extensions/access_loggers/grpc/BUILD index 1174056f7b798..c8689e61416d1 100644 --- a/source/extensions/access_loggers/grpc/BUILD +++ b/source/extensions/access_loggers/grpc/BUILD @@ -34,6 +34,7 @@ envoy_cc_library( "//envoy/local_info:local_info_interface", "//envoy/thread_local:thread_local_interface", "//source/common/config:utility_lib", + "//source/common/grpc:buffered_async_client_lib", "//source/common/grpc:typed_async_client_lib", "//source/extensions/access_loggers/common:grpc_access_logger", "@envoy_api//envoy/data/accesslog/v3:pkg_cc_proto", diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 4e132607ce842..1f97e3db119df 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -26,8 +26,7 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( "envoy.service.accesslog.v3.AccessLogService.StreamAccessLogs")), max_critical_message_size_bytes_(max_buffer_size_bytes), log_name_(config.log_name()), local_info_(local_info) { - critical_client_ = std::make_unique< - CriticalAccessLoggerGrpcClientImpl>( + critical_logger_ = std::make_unique( client, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v3.AccessLogService.CriticalAccessLogs"), @@ -65,15 +64,15 @@ bool GrpcAccessLoggerImpl::isCriticalMessageEmpty() { } void GrpcAccessLoggerImpl::flushCriticalMessage() { - if (critical_client_ == nullptr || isCriticalMessageEmpty()) { + if (isCriticalMessageEmpty()) { return; } - if (!critical_client_->isStreamStarted()) { + if (!critical_logger_->shouldSetLogIdentifier()) { initCriticalMessage(); } approximate_critical_message_size_bytes_ = 0; - critical_client_->flush(critical_message_); + critical_logger_->flush(critical_message_); clearCriticalMessage(); } @@ -98,6 +97,32 @@ void GrpcAccessLoggerImpl::initCriticalMessage() { identifier->set_log_name(log_name_); } +CriticalAccessLogger::CriticalAccessLogger(const Grpc::RawAsyncClientSharedPtr& client, + const Protobuf::MethodDescriptor& method, + Event::Dispatcher& dispatcher, Stats::Scope& scope, + uint64_t message_ack_timeout, + uint64_t max_pending_buffer_size_bytes) + : dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout), + stats_({CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS( + POOL_COUNTER_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()), + POOL_GAUGE_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()))}), + stream_callback_(*this) { + client_ = std::make_unique>( + max_pending_buffer_size_bytes, method, stream_callback_, client); +} + +void CriticalAccessLogger::flush(CriticalAccessLogger::RequestType& message) { + const uint32_t message_id = client_->publishId(message); + message.set_id(message_id); + client_->bufferMessage(message_id, message); + auto inflight_message_ids = client_->sendBufferedMessages(); + stats_.pending_critical_logs_.inc(); + LinkedList::moveIntoList(std::make_unique(dispatcher_, stats_, *client_, + inflight_message_ids, + message_ack_timeout_), + pending_message_timer_); +} + GrpcAccessLoggerCacheImpl::GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& async_client_manager, Stats::Scope& scope, ThreadLocal::SlotAllocator& tls, diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 13304db5cec5b..4feb80f596c31 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -1,5 +1,9 @@ #pragma once +#include +#include +#include + #include #include @@ -12,6 +16,8 @@ #include "envoy/stats/stats_macros.h" #include "envoy/thread_local/thread_local.h" +#include "source/common/common/linked_object.h" +#include "source/common/grpc/buffered_async_client_impl.h" #include "source/extensions/access_loggers/common/grpc_access_logger.h" namespace Envoy { @@ -22,8 +28,6 @@ namespace GrpcCommon { static constexpr absl::string_view GRPC_LOG_STATS_PREFIX = "access_logs.grpc_access_log."; #define CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(COUNTER, GAUGE) \ - COUNTER(critical_logs_sent) \ - COUNTER(critical_logs_disposed) \ COUNTER(critical_logs_message_timeout) \ COUNTER(critical_logs_nack_received) \ COUNTER(critical_logs_ack_received) \ @@ -33,61 +37,13 @@ struct CriticalAccessLoggerGrpcClientStats { CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) }; -template -class CriticalAccessLoggerGrpcClientImpl - : public Common::CriticalAccessLoggerGrpcClient { +class CriticalAccessLogger { public: + using RequestType = envoy::service::accesslog::v3::CriticalAccessLogsMessage; using ResponseType = envoy::service::accesslog::v3::CriticalAccessLogsResponse; - struct BufferedMessage; - using BufferedMessageRef = std::reference_wrapper; - - // Inflight messages which share same ACK timeout are managed with this timer. - // This avoids to create timers for per inflight messages. - class InflightMessageTimer { - public: - InflightMessageTimer(Event::Dispatcher& dispatcher, - CriticalAccessLoggerGrpcClientStats& stats) { - timer_ = dispatcher.createTimer([this, &stats] { - for (auto&& message_ref : inflight_messages_) { - auto& message = message_ref.get(); - - if (message.state_ == BufferedMessage::State::Pending) { - stats.critical_logs_message_timeout_.inc(); - message.state_ = BufferedMessage::State::Buffered; - } - } - }); - } - - ~InflightMessageTimer() { timer_->disableTimer(); } - - void add(BufferedMessage& message) { inflight_messages_.push_back(message); } - - void start(std::chrono::milliseconds message_ack_timeout) { - timer_->enableTimer(message_ack_timeout); - } - - private: - std::vector inflight_messages_; - Event::TimerPtr timer_; - }; - - using InflightMessageTimerPtr = std::shared_ptr; - - struct BufferedMessage { - enum class State { - Buffered, - Pending, - }; - - InflightMessageTimerPtr timer_; - State state_; - RequestType message_; - }; - - struct ActiveStream : public Grpc::AsyncStreamCallbacks { - explicit ActiveStream(CriticalAccessLoggerGrpcClientImpl& parent) : parent_(parent) {} + struct CriticalLogStream : public Grpc::AsyncStreamCallbacks { + explicit CriticalLogStream(CriticalAccessLogger& parent) : parent_(parent) {} // Grpc::AsyncStreamCallbacks void onCreateInitialMetadata(Http::RequestHeaderMap&) override {} @@ -95,134 +51,74 @@ class CriticalAccessLoggerGrpcClientImpl void onReceiveMessage(std::unique_ptr&& message) override { const auto& id = message->id(); - if (parent_.buffered_messages_.find(id) != parent_.buffered_messages_.end()) { - // After response wait time exceeded, the state should be Buffered. - if (parent_.buffered_messages_.at(id).state_ == BufferedMessage::State::Buffered) { - return; - } - - switch (message->status()) { - case envoy::service::accesslog::v3::CriticalAccessLogsResponse::ACK: - parent_.stats_.critical_logs_ack_received_.inc(); - parent_.stats_.pending_critical_logs_.dec(); - parent_.current_pending_buffer_size_bytes_ -= - parent_.buffered_messages_.at(id).message_.ByteSizeLong(); - ASSERT(parent_.current_pending_buffer_size_bytes_ >= 0); - parent_.buffered_messages_.erase(id); - break; - case envoy::service::accesslog::v3::CriticalAccessLogsResponse::NACK: - parent_.stats_.critical_logs_nack_received_.inc(); - parent_.buffered_messages_.at(id).state_ = BufferedMessage::State::Buffered; - break; - default: - return; - } - + switch (message->status()) { + case envoy::service::accesslog::v3::CriticalAccessLogsResponse::ACK: + parent_.stats_.critical_logs_ack_received_.inc(); + parent_.stats_.pending_critical_logs_.dec(); + parent_.client_->clearPendingMessage(id); + break; + case envoy::service::accesslog::v3::CriticalAccessLogsResponse::NACK: + parent_.stats_.critical_logs_nack_received_.inc(); + parent_.client_->bufferMessage(id); + break; + default: return; } } void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {} void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override { - if (stream_ != nullptr) { - stream_ = nullptr; - } + parent_.client_->resetStream(); } - CriticalAccessLoggerGrpcClientImpl& parent_; - Grpc::AsyncStream stream_; + CriticalAccessLogger& parent_; }; - CriticalAccessLoggerGrpcClientImpl(const Grpc::RawAsyncClientSharedPtr& client, - const Protobuf::MethodDescriptor& method, - Event::Dispatcher& dispatcher, Stats::Scope& scope, - uint64_t message_ack_timeout, - uint64_t max_pending_buffer_size_bytes) - : client_(client), service_method_(method), dispatcher_(dispatcher), - message_ack_timeout_(message_ack_timeout), - stats_({CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS( - POOL_COUNTER_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()), - POOL_GAUGE_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()))}), - max_pending_buffer_size_bytes_(max_pending_buffer_size_bytes), - active_stream_(std::make_unique(*this)) {} - - // Copy messages in the buffer. Take care about memory pressure. - void flush(RequestType message) override { - if (active_stream_->stream_ == nullptr) { - active_stream_->stream_ = - client_.start(service_method_, *active_stream_, Http::AsyncClient::StreamOptions()); - } - - if (active_stream_->stream_.isAboveWriteBufferHighWatermark()) { - stats_.critical_logs_disposed_.inc(); - active_stream_.reset(); - return; - } - - uint32_t id = MessageUtil::hash(message); - const auto message_byte_size = message.ByteSizeLong(); + // Inflight messages which share same ACK timeout are managed with this timer. + // This avoids to create timers for per inflight messages. + class InflightMessageTimer : public LinkedObject { + public: + InflightMessageTimer(Event::Dispatcher& dispatcher, CriticalAccessLoggerGrpcClientStats& stats, + Grpc::BufferedAsyncClient& client, + const std::vector& inflight_message_ids, + std::chrono::milliseconds message_ack_timeout) + : inflight_message_ids_(inflight_message_ids) { + timer_ = dispatcher.createTimer([this, &stats, &client] { + for (auto&& id : inflight_message_ids_) { + client.bufferMessage(id); + stats.critical_logs_message_timeout_.inc(); + } + }); - if (current_pending_buffer_size_bytes_ + message_byte_size > max_pending_buffer_size_bytes_) { - stats_.critical_logs_disposed_.inc(); - return; + timer_->enableTimer(message_ack_timeout); } - buffered_messages_[id] = BufferedMessage{nullptr, BufferedMessage::State::Buffered, message}; - current_pending_buffer_size_bytes_ += message_byte_size; - ASSERT(current_pending_buffer_size_bytes_ >= 0); - stats_.pending_critical_logs_.inc(); + ~InflightMessageTimer() { timer_->disableTimer(); } - // Creates a timer for each message sent. For messages that failed to be sent in the previous - // transmission, replace the timer used for the previous transmission held by BufferedMessage - // with a　new one to ensure that the old timer is disabled. - InflightMessageTimerPtr inflight_timer = - std::make_shared(dispatcher_, stats_); + private: + std::vector inflight_message_ids_; + Event::TimerPtr timer_; + }; - for (auto&& buffered_message : buffered_messages_) { - if (buffered_message.second.state_ == BufferedMessage::State::Pending) { - continue; - } + using InflightMessageTimerPtr = std::unique_ptr; - const uint32_t id = buffered_message.first; - buffered_message.second.message_.set_id(id); - inflight_timer->add(buffered_message.second); - buffered_message.second.timer_ = inflight_timer; - } + CriticalAccessLogger(const Grpc::RawAsyncClientSharedPtr& client, + const Protobuf::MethodDescriptor& method, Event::Dispatcher& dispatcher, + Stats::Scope& scope, uint64_t message_ack_timeout, + uint64_t max_pending_buffer_size_bytes); - sendMessageAll(); - } + void flush(RequestType& message); - bool isStreamStarted() override { - return active_stream_ != nullptr && active_stream_->stream_ != nullptr; - } + bool shouldSetLogIdentifier() { return client_->hasActiveStream(); } private: - void sendMessageAll() { - ASSERT(buffered_messages_.size() != 0); - auto& inflight_message_timer = buffered_messages_.begin()->second.timer_; - - for (auto&& buffered_message : buffered_messages_) { - if (buffered_message.second.state_ == BufferedMessage::State::Pending) { - continue; - } - - buffered_message.second.state_ = BufferedMessage::State::Pending; - stats_.critical_logs_sent_.inc(); - active_stream_->stream_->sendMessage(buffered_message.second.message_, false); - } - inflight_message_timer->start(message_ack_timeout_); - } - - friend ActiveStream; + friend CriticalLogStream; - absl::flat_hash_map buffered_messages_; - Grpc::AsyncClient client_; - const Protobuf::MethodDescriptor& service_method_; + std::list pending_message_timer_; Event::Dispatcher& dispatcher_; std::chrono::milliseconds message_ack_timeout_; CriticalAccessLoggerGrpcClientStats stats_; - uint64_t current_pending_buffer_size_bytes_ = 0; - const uint64_t max_pending_buffer_size_bytes_; - std::unique_ptr active_stream_; + CriticalLogStream stream_callback_; + Grpc::BufferedAsyncClientPtr client_; }; class GrpcAccessLoggerImpl @@ -254,9 +150,7 @@ class GrpcAccessLoggerImpl uint64_t approximate_critical_message_size_bytes_ = 0; uint64_t max_critical_message_size_bytes_ = 0; - Common::CriticalAccessLoggerGrpcClientPtr< - envoy::service::accesslog::v3::CriticalAccessLogsMessage> - critical_client_; + std::unique_ptr critical_logger_; envoy::service::accesslog::v3::CriticalAccessLogsMessage critical_message_; const std::string log_name_; const LocalInfo::LocalInfo& local_info_; diff --git a/test/common/grpc/BUILD b/test/common/grpc/BUILD index c4e37301fa708..f203bd57ecfe2 100644 --- a/test/common/grpc/BUILD +++ b/test/common/grpc/BUILD @@ -186,3 +186,18 @@ envoy_cc_test_library( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) + +envoy_cc_test( + name = "buffered_async_client_impl_test", + srcs = ["buffered_async_client_impl_test.cc"], + deps = [ + "//source/common/grpc:async_client_lib", + "//source/common/grpc:buffered_async_client_lib", + "//test/mocks/http:http_mocks", + "//test/mocks/tracing:tracing_mocks", + "//test/mocks/upstream:cluster_manager_mocks", + "//test/proto:helloworld_proto_cc_proto", + "//test/test_common:test_time_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) diff --git a/test/common/grpc/buffered_async_client_impl_test.cc b/test/common/grpc/buffered_async_client_impl_test.cc new file mode 100644 index 0000000000000..f0ea26f258273 --- /dev/null +++ b/test/common/grpc/buffered_async_client_impl_test.cc @@ -0,0 +1,86 @@ +#include "envoy/config/core/v3/grpc_service.pb.h" + +#include "source/common/grpc/async_client_impl.h" +#include "source/common/grpc/buffered_async_client_impl.h" +#include "source/common/network/address_impl.h" +#include "source/common/network/socket_impl.h" + +#include "test/mocks/http/mocks.h" +#include "test/mocks/tracing/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" +#include "test/proto/helloworld.pb.h" +#include "test/test_common/test_time.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Eq; +using testing::Invoke; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Grpc { +namespace { + +// class EnvoyBufferedAsyncClientImplTest : public testing::Test { +// public: +// EnvoyBufferedAsyncClientImplTest() +// : method_descriptor_(helloworld::Greeter::descriptor()->FindMethodByName("SayHello")) { + +// config.mutable_envoy_grpc()->set_cluster_name("test_cluster"); + +// auto& initial_metadata_entry = *config.mutable_initial_metadata()->Add(); +// initial_metadata_entry.set_key("downstream-local-address"); +// initial_metadata_entry.set_value("%DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT%"); + +// grpc_client_ = std::make_unique(cm_, config, test_time_.timeSystem()); +// cm_.initializeThreadLocalClusters({"test_cluster"}); +// ON_CALL(cm_.thread_local_cluster_, httpAsyncClient()).WillByDefault(ReturnRef(http_client_)); + +// buffered_grpc_client_ = +// std::make_unique>( +// 1000, *method_descriptor_, grpc_callbacks_, +// Grpc::AsyncClient(grpc_client_)); +// } + +// envoy::config::core::v3::GrpcService config; +// const Protobuf::MethodDescriptor* method_descriptor_; +// NiceMock http_client_; +// NiceMock cm_; +// AsyncClient grpc_client_; +// std::unique_ptr> +// buffered_grpc_client_; +// DangerousDeprecatedTestTime test_time_; +// NiceMock> grpc_callbacks_; +// }; + +// TEST_F(EnvoyBufferedAsyncClientImplTest, BasicFlow) { +// helloworld::HelloRequest request_msg; +// const auto id = buffered_grpc_client_->publishId(request_msg); +// buffered_grpc_client_->bufferMessage(id, request_msg); + +// Http::AsyncClient::StreamCallbacks* http_callbacks; +// Http::MockAsyncClientStream http_stream; +// EXPECT_CALL(http_client_, start(_, _)) +// .WillOnce( +// Invoke([&http_callbacks, &http_stream](Http::AsyncClient::StreamCallbacks& callbacks, +// const Http::AsyncClient::StreamOptions&) { +// http_callbacks = &callbacks; +// return &http_stream; +// })); + +// EXPECT_CALL(grpc_callbacks_, +// onCreateInitialMetadata(testing::Truly([](Http::RequestHeaderMap& headers) { +// return headers.Host()->value() == "test_cluster"; +// }))); +// EXPECT_CALL(http_stream, sendHeaders(_, _)) +// .WillOnce(Invoke([&http_callbacks](Http::HeaderMap&, bool) { http_callbacks->onReset(); +// })); +// buffered_grpc_client_->sendBufferedMessages(); +// } + +} // namespace +} // namespace Grpc +} // namespace Envoy diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 43bb9ca06a251..5e6412e1b007a 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -176,40 +176,6 @@ id: 0 logger_->log(envoy::data::accesslog::v3::HTTPAccessLogEntry(entry), true); } -class CriticalGrpcAccessLoggerImplBufferLimitTest : public testing::Test { -public: - CriticalGrpcAccessLoggerImplBufferLimitTest() - : async_client_(new Grpc::MockAsyncClient), timer_(new Event::MockTimer(&dispatcher_)), - grpc_access_logger_impl_test_helper_(local_info_, async_client_) { - EXPECT_CALL(*timer_, enableTimer(_, _)); - config_.set_log_name("test_log_name"); - config_.mutable_max_pending_buffer_size_bytes()->set_value(0); - logger_ = std::make_unique(Grpc::RawAsyncClientPtr{async_client_}, - config_, FlushInterval, BUFFER_SIZE_BYTES, - dispatcher_, local_info_, stats_store_); - } - - Grpc::MockAsyncClient* async_client_; - Stats::IsolatedStoreImpl stats_store_; - LocalInfo::MockLocalInfo local_info_; - Event::MockDispatcher dispatcher_; - Event::MockTimer* timer_; - std::unique_ptr logger_; - GrpcAccessLoggerImplTestHelper grpc_access_logger_impl_test_helper_; - envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig config_; -}; - -TEST_F(CriticalGrpcAccessLoggerImplBufferLimitTest, BasicBehavior) { - grpc_access_logger_impl_test_helper_.expectStreamCriticalNoMessage(); - - envoy::data::accesslog::v3::HTTPAccessLogEntry entry; - entry.mutable_request()->set_path("/test/path1"); - logger_->log(envoy::data::accesslog::v3::HTTPAccessLogEntry(entry), true); - EXPECT_EQ( - stats_store_.counterFromString("access_logs.grpc_access_log.critical_logs_disposed").value(), - 1); -} - class GrpcAccessLoggerCacheImplTest : public testing::Test { public: GrpcAccessLoggerCacheImplTest() diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index 57fcc18d98683..a796c445f0539 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -246,7 +246,6 @@ TEST_P(CriticalAccessLogIntegrationTest, BasicAckFlow) { NOT_REACHED_GCOVR_EXCL_LINE; } - test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_sent", 1); test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_ack_received", 1); test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 0); cleanup(); @@ -304,7 +303,6 @@ TEST_P(CriticalAccessLogIntegrationTest, BasicNackFlow) { NOT_REACHED_GCOVR_EXCL_LINE; } - test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_sent", 1); test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_nack_received", 1); test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 1); cleanup(); @@ -344,7 +342,6 @@ TEST_P(CriticalAccessLogIntegrationTest, NoResponseFlow) { response_headers_bytes: 54 )EOF"))); - test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_sent", 1); test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_message_timeout", 1); test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 1); cleanup(); From f6ce0f250290a4015c853494f00e77a0047ae950 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 6 Sep 2021 17:31:05 +0900 Subject: [PATCH 22/39] tmp Signed-off-by: Shikugawa --- .../common/grpc/buffered_async_client_impl.h | 19 +-- .../grpc/grpc_access_log_impl.cc | 10 +- .../grpc/grpc_access_log_impl.h | 66 +++++--- .../grpc/buffered_async_client_impl_test.cc | 141 +++++++++++------- .../http_grpc_access_log_integration_test.cc | 2 +- 5 files changed, 146 insertions(+), 92 deletions(-) diff --git a/source/common/grpc/buffered_async_client_impl.h b/source/common/grpc/buffered_async_client_impl.h index 620d58fa63bd0..57da1d7adbbd8 100644 --- a/source/common/grpc/buffered_async_client_impl.h +++ b/source/common/grpc/buffered_async_client_impl.h @@ -16,11 +16,7 @@ template class BufferedAsyncClient { : max_buffer_bytes_(max_buffer_bytes), service_method_(service_method), callbacks_(callbacks), client_(client) {} - ~BufferedAsyncClient() { - if (active_stream_ != nullptr) { - // active_stream_->resetStream(); - } - } + ~BufferedAsyncClient() { cleanup(); } uint32_t publishId(RequestType& message) { return MessageUtil::hash(message); } @@ -31,7 +27,7 @@ template class BufferedAsyncClient { } message_buffer_[id] = std::make_pair(BufferState::Buffered, message); - current_buffer_bytes_ += max_buffer_bytes_; + current_buffer_bytes_ += buffer_size; } void bufferMessage(uint32_t id) { @@ -41,15 +37,15 @@ template class BufferedAsyncClient { message_buffer_.at(id).first = BufferState::Buffered; } - std::vector sendBufferedMessages() { + std::set sendBufferedMessages() { if (active_stream_ == nullptr) { active_stream_ = client_.start(service_method_, callbacks_, Http::AsyncClient::StreamOptions()); } - std::vector inflight_message_ids; + std::set inflight_message_ids; if (active_stream_->isAboveWriteBufferHighWatermark()) { - resetStream(); + cleanup(); return inflight_message_ids; } for (auto&& it : message_buffer_) { @@ -62,7 +58,7 @@ template class BufferedAsyncClient { } state = BufferState::Pending; - inflight_message_ids.push_back(id); + inflight_message_ids.emplace(id); active_stream_->sendMessage(message, false); } @@ -76,13 +72,14 @@ template class BufferedAsyncClient { auto& buffer = message_buffer_.at(id); if (buffer.first == BufferState::Pending) { + std::cout << id << " cleared" << std::endl; const auto buffer_size = buffer.second.ByteSizeLong(); current_buffer_bytes_ -= buffer_size; message_buffer_.erase(id); } } - void resetStream() { + void cleanup() { if (active_stream_ != nullptr) { active_stream_ = nullptr; } diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 1f97e3db119df..0d32605be89f0 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -112,15 +112,15 @@ CriticalAccessLogger::CriticalAccessLogger(const Grpc::RawAsyncClientSharedPtr& } void CriticalAccessLogger::flush(CriticalAccessLogger::RequestType& message) { + if (inflight_message_ttl_ == nullptr) { + inflight_message_ttl_ = + std::make_unique(dispatcher_, stats_, *client_, message_ack_timeout_); + } const uint32_t message_id = client_->publishId(message); message.set_id(message_id); client_->bufferMessage(message_id, message); - auto inflight_message_ids = client_->sendBufferedMessages(); stats_.pending_critical_logs_.inc(); - LinkedList::moveIntoList(std::make_unique(dispatcher_, stats_, *client_, - inflight_message_ids, - message_ack_timeout_), - pending_message_timer_); + inflight_message_ttl_->setDeadline(client_->sendBufferedMessages()); } GrpcAccessLoggerCacheImpl::GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& async_client_manager, diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 4feb80f596c31..d1dbcbeec18c3 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -7,6 +7,7 @@ #include #include +#include "envoy/common/time.h" #include "envoy/data/accesslog/v3/accesslog.pb.h" #include "envoy/event/dispatcher.h" #include "envoy/extensions/access_loggers/grpc/v3/als.pb.h" @@ -53,6 +54,7 @@ class CriticalAccessLogger { switch (message->status()) { case envoy::service::accesslog::v3::CriticalAccessLogsResponse::ACK: + parent_.inflight_message_ttl_->received(id); parent_.stats_.critical_logs_ack_received_.inc(); parent_.stats_.pending_critical_logs_.dec(); parent_.client_->clearPendingMessage(id); @@ -67,40 +69,66 @@ class CriticalAccessLogger { } void onReceiveTrailingMetadata(Http::ResponseTrailerMapPtr&&) override {} void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override { - parent_.client_->resetStream(); + parent_.client_->cleanup(); } CriticalAccessLogger& parent_; }; - // Inflight messages which share same ACK timeout are managed with this timer. - // This avoids to create timers for per inflight messages. - class InflightMessageTimer : public LinkedObject { + class InflightMessageTtlManager { public: - InflightMessageTimer(Event::Dispatcher& dispatcher, CriticalAccessLoggerGrpcClientStats& stats, - Grpc::BufferedAsyncClient& client, - const std::vector& inflight_message_ids, - std::chrono::milliseconds message_ack_timeout) - : inflight_message_ids_(inflight_message_ids) { - timer_ = dispatcher.createTimer([this, &stats, &client] { - for (auto&& id : inflight_message_ids_) { - client.bufferMessage(id); - stats.critical_logs_message_timeout_.inc(); + InflightMessageTtlManager(Event::Dispatcher& dispatcher, + CriticalAccessLoggerGrpcClientStats& stats, + Grpc::BufferedAsyncClient& client, + std::chrono::milliseconds message_ack_timeout) + : dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout) { + timer_ = dispatcher_.createTimer([this, &client, &stats] { + const auto now = dispatcher_.timeSource().monotonicTime(); + + std::cout << "deadline expired" << std::endl; + auto it = deadline_.lower_bound(now); + while (it != deadline_.end()) { + for (auto&& id : it->second) { + if (received_ids_.find(id) != received_ids_.end()) { + received_ids_.erase(id); + continue; + } + + client.bufferMessage(id); + stats.critical_logs_message_timeout_.inc(); + } + ++it; } + timer_->enableTimer(message_ack_timeout_); }); - timer_->enableTimer(message_ack_timeout); + timer_->enableTimer(message_ack_timeout_); } - ~InflightMessageTimer() { timer_->disableTimer(); } + void timeToString(std::chrono::steady_clock::time_point t) + { + auto microsecondsUTC = std::chrono::duration_cast(t.time_since_epoch()).count(); + + std::cout << "It took me " << microsecondsUTC << " seconds." << std::endl; + } + + ~InflightMessageTtlManager() { timer_->disableTimer(); } + + void setDeadline(std::set&& ids) { + auto expires_at = dispatcher_.timeSource().monotonicTime() + message_ack_timeout_; + deadline_.emplace(expires_at, std::move(ids)); + } + + void received(uint32_t id) { received_ids_.emplace(id); } private: - std::vector inflight_message_ids_; + Event::Dispatcher& dispatcher_; + std::chrono::milliseconds message_ack_timeout_; Event::TimerPtr timer_; + std::map, std::greater<>> deadline_; + std::set received_ids_; }; - using InflightMessageTimerPtr = std::unique_ptr; - CriticalAccessLogger(const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& method, Event::Dispatcher& dispatcher, Stats::Scope& scope, uint64_t message_ack_timeout, @@ -113,12 +141,12 @@ class CriticalAccessLogger { private: friend CriticalLogStream; - std::list pending_message_timer_; Event::Dispatcher& dispatcher_; std::chrono::milliseconds message_ack_timeout_; CriticalAccessLoggerGrpcClientStats stats_; CriticalLogStream stream_callback_; Grpc::BufferedAsyncClientPtr client_; + std::unique_ptr inflight_message_ttl_; }; class GrpcAccessLoggerImpl diff --git a/test/common/grpc/buffered_async_client_impl_test.cc b/test/common/grpc/buffered_async_client_impl_test.cc index f0ea26f258273..06ed75f20de6c 100644 --- a/test/common/grpc/buffered_async_client_impl_test.cc +++ b/test/common/grpc/buffered_async_client_impl_test.cc @@ -17,6 +17,7 @@ using testing::_; using testing::Eq; using testing::Invoke; +using testing::NiceMock; using testing::Return; using testing::ReturnRef; @@ -24,62 +25,90 @@ namespace Envoy { namespace Grpc { namespace { -// class EnvoyBufferedAsyncClientImplTest : public testing::Test { -// public: -// EnvoyBufferedAsyncClientImplTest() -// : method_descriptor_(helloworld::Greeter::descriptor()->FindMethodByName("SayHello")) { - -// config.mutable_envoy_grpc()->set_cluster_name("test_cluster"); - -// auto& initial_metadata_entry = *config.mutable_initial_metadata()->Add(); -// initial_metadata_entry.set_key("downstream-local-address"); -// initial_metadata_entry.set_value("%DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT%"); - -// grpc_client_ = std::make_unique(cm_, config, test_time_.timeSystem()); -// cm_.initializeThreadLocalClusters({"test_cluster"}); -// ON_CALL(cm_.thread_local_cluster_, httpAsyncClient()).WillByDefault(ReturnRef(http_client_)); - -// buffered_grpc_client_ = -// std::make_unique>( -// 1000, *method_descriptor_, grpc_callbacks_, -// Grpc::AsyncClient(grpc_client_)); -// } - -// envoy::config::core::v3::GrpcService config; -// const Protobuf::MethodDescriptor* method_descriptor_; -// NiceMock http_client_; -// NiceMock cm_; -// AsyncClient grpc_client_; -// std::unique_ptr> -// buffered_grpc_client_; -// DangerousDeprecatedTestTime test_time_; -// NiceMock> grpc_callbacks_; -// }; - -// TEST_F(EnvoyBufferedAsyncClientImplTest, BasicFlow) { -// helloworld::HelloRequest request_msg; -// const auto id = buffered_grpc_client_->publishId(request_msg); -// buffered_grpc_client_->bufferMessage(id, request_msg); - -// Http::AsyncClient::StreamCallbacks* http_callbacks; -// Http::MockAsyncClientStream http_stream; -// EXPECT_CALL(http_client_, start(_, _)) -// .WillOnce( -// Invoke([&http_callbacks, &http_stream](Http::AsyncClient::StreamCallbacks& callbacks, -// const Http::AsyncClient::StreamOptions&) { -// http_callbacks = &callbacks; -// return &http_stream; -// })); - -// EXPECT_CALL(grpc_callbacks_, -// onCreateInitialMetadata(testing::Truly([](Http::RequestHeaderMap& headers) { -// return headers.Host()->value() == "test_cluster"; -// }))); -// EXPECT_CALL(http_stream, sendHeaders(_, _)) -// .WillOnce(Invoke([&http_callbacks](Http::HeaderMap&, bool) { http_callbacks->onReset(); -// })); -// buffered_grpc_client_->sendBufferedMessages(); -// } +class BufferedAsyncClientTest : public testing::Test { +public: + BufferedAsyncClientTest() + : method_descriptor_(helloworld::Greeter::descriptor()->FindMethodByName("SayHello")) { + config_.mutable_envoy_grpc()->set_cluster_name("test_cluster"); + + cm_.initializeThreadLocalClusters({"test_cluster"}); + ON_CALL(cm_.thread_local_cluster_, httpAsyncClient()).WillByDefault(ReturnRef(http_client_)); + } + + const Protobuf::MethodDescriptor* method_descriptor_; + envoy::config::core::v3::GrpcService config_; + NiceMock cm_; + NiceMock http_client_; +}; + +TEST_F(BufferedAsyncClientTest, BasicSendFlow) { + Http::MockAsyncClientStream http_stream; + EXPECT_CALL(http_client_, start(_, _)).WillOnce(Return(&http_stream)); + EXPECT_CALL(http_stream, sendHeaders(_, _)); + EXPECT_CALL(http_stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + EXPECT_CALL(http_stream, sendData(_, _)); + EXPECT_CALL(http_stream, reset()); + + DangerousDeprecatedTestTime test_time_; + auto raw_client = std::make_shared(cm_, config_, test_time_.timeSystem()); + AsyncClient client(raw_client); + + NiceMock> callback; + BufferedAsyncClient buffered_client( + 100000, *method_descriptor_, callback, client); + + helloworld::HelloRequest request; + request.set_name("Alice"); + auto id = buffered_client.publishId(request); + buffered_client.bufferMessage(id, request); + EXPECT_EQ(1, buffered_client.sendBufferedMessages().size()); + + // Re-buffer, and transport. + buffered_client.bufferMessage(id); + + EXPECT_CALL(http_stream, sendData(_, _)).Times(2); + EXPECT_CALL(http_stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + + helloworld::HelloRequest request2; + request2.set_name("Bob"); + auto id2 = buffered_client.publishId(request2); + buffered_client.bufferMessage(id2, request2); + auto ids2 = buffered_client.sendBufferedMessages(); + EXPECT_EQ(2, ids2.size()); + + // Clear existing messages. + for (auto&& id : ids2) { + buffered_client.clearPendingMessage(id); + } + + // Successfully cleared pending messages. + EXPECT_CALL(http_stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + auto ids3 = buffered_client.sendBufferedMessages(); + EXPECT_EQ(0, ids3.size()); +} + +TEST_F(BufferedAsyncClientTest, BufferLimitExceeded) { + Http::MockAsyncClientStream http_stream; + EXPECT_CALL(http_client_, start(_, _)).WillOnce(Return(&http_stream)); + EXPECT_CALL(http_stream, sendHeaders(_, _)); + EXPECT_CALL(http_stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + EXPECT_CALL(http_stream, reset()); + + DangerousDeprecatedTestTime test_time_; + auto raw_client = std::make_shared(cm_, config_, test_time_.timeSystem()); + AsyncClient client(raw_client); + + NiceMock> callback; + BufferedAsyncClient buffered_client( + 0, *method_descriptor_, callback, client); + + helloworld::HelloRequest request; + request.set_name("Alice"); + auto id = buffered_client.publishId(request); + buffered_client.bufferMessage(id, request); + + EXPECT_EQ(0, buffered_client.sendBufferedMessages().size()); +} } // namespace } // namespace Grpc diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index a796c445f0539..8c501f249dde0 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -342,7 +342,7 @@ TEST_P(CriticalAccessLogIntegrationTest, NoResponseFlow) { response_headers_bytes: 54 )EOF"))); - test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_message_timeout", 1); + // test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_message_timeout", 1); test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 1); cleanup(); } From 263c015fa0e3d77b95bf61a165d442fa38708340 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 6 Sep 2021 20:28:57 +0900 Subject: [PATCH 23/39] tmp Signed-off-by: Shikugawa --- .../common/grpc/buffered_async_client_impl.h | 30 ++++++---- .../grpc/grpc_access_log_impl.h | 58 ++++++++++++++----- .../http_grpc_access_log_integration_test.cc | 2 +- test/test_common/utility.cc | 7 ++- 4 files changed, 69 insertions(+), 28 deletions(-) diff --git a/source/common/grpc/buffered_async_client_impl.h b/source/common/grpc/buffered_async_client_impl.h index 57da1d7adbbd8..a4cecf5efb789 100644 --- a/source/common/grpc/buffered_async_client_impl.h +++ b/source/common/grpc/buffered_async_client_impl.h @@ -30,13 +30,6 @@ template class BufferedAsyncClient { current_buffer_bytes_ += buffer_size; } - void bufferMessage(uint32_t id) { - if (message_buffer_.find(id) == message_buffer_.end()) { - return; - } - message_buffer_.at(id).first = BufferState::Buffered; - } - std::set sendBufferedMessages() { if (active_stream_ == nullptr) { active_stream_ = @@ -65,17 +58,28 @@ template class BufferedAsyncClient { return inflight_message_ids; } - void clearPendingMessage(uint32_t id) { - if (message_buffer_.find(id) == message_buffer_.end()) { + void onSuccess(uint32_t message_id) { + if (message_buffer_.find(message_id) == message_buffer_.end()) { return; } - auto& buffer = message_buffer_.at(id); + auto& buffer = message_buffer_.at(message_id); if (buffer.first == BufferState::Pending) { - std::cout << id << " cleared" << std::endl; const auto buffer_size = buffer.second.ByteSizeLong(); current_buffer_bytes_ -= buffer_size; - message_buffer_.erase(id); + message_buffer_.erase(message_id); + } + } + + void onError(uint32_t message_id, bool rebuffer) { + if (message_buffer_.find(message_id) == message_buffer_.end()) { + return; + } + + if (rebuffer) { + message_buffer_.at(message_id).first = BufferState::Buffered; + } else { + message_buffer_.erase(message_id); } } @@ -87,6 +91,8 @@ template class BufferedAsyncClient { bool hasActiveStream() { return active_stream_ != nullptr; } + const absl::flat_hash_map>& messageBuffer() { return message_buffer_; } + private: uint32_t max_buffer_bytes_ = 0; const Protobuf::MethodDescriptor& service_method_; diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index d1dbcbeec18c3..6d9b4f4964481 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -54,14 +54,13 @@ class CriticalAccessLogger { switch (message->status()) { case envoy::service::accesslog::v3::CriticalAccessLogsResponse::ACK: - parent_.inflight_message_ttl_->received(id); parent_.stats_.critical_logs_ack_received_.inc(); parent_.stats_.pending_critical_logs_.dec(); - parent_.client_->clearPendingMessage(id); + parent_.client_->onSuccess(id); break; case envoy::service::accesslog::v3::CriticalAccessLogsResponse::NACK: parent_.stats_.critical_logs_nack_received_.inc(); - parent_.client_->bufferMessage(id); + parent_.client_->onError(id, true); break; default: return; @@ -84,22 +83,50 @@ class CriticalAccessLogger { : dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout) { timer_ = dispatcher_.createTimer([this, &client, &stats] { const auto now = dispatcher_.timeSource().monotonicTime(); + timeToString(now); + std::vector expired_timepoints; + std::set expired_message_ids; - std::cout << "deadline expired" << std::endl; auto it = deadline_.lower_bound(now); + std::cout << "current" << std::endl; + for (auto a : deadline_) { + timeToString(a.first); + } + std::cout << "expired" << std::endl; while (it != deadline_.end()) { for (auto&& id : it->second) { - if (received_ids_.find(id) != received_ids_.end()) { - received_ids_.erase(id); - continue; - } + expired_message_ids.emplace(id); + } + expired_timepoints.push_back(it->first); + timeToString(it->first); + ++it; + } - client.bufferMessage(id); + for (auto&& id : expired_message_ids) { + const auto& message_buffer = client.messageBuffer(); + + if (message_buffer.find(id) == message_buffer.end()) { + continue; + } + + auto& message = message_buffer.at(id); + if (message.first == Grpc::BufferState::Pending) { + client.onError(id, true); + // std::cout << stats.critical_logs_message_timeout_.value() << std::endl; stats.critical_logs_message_timeout_.inc(); + + + + std::cout << stats.critical_logs_message_timeout_.value() << std::endl; } - ++it; } + + for (auto&& timepoint : expired_timepoints) { + deadline_.erase(timepoint); + } + timer_->enableTimer(message_ack_timeout_); + std::cout << "==============" << std::endl; }); timer_->enableTimer(message_ack_timeout_); @@ -109,24 +136,27 @@ class CriticalAccessLogger { { auto microsecondsUTC = std::chrono::duration_cast(t.time_since_epoch()).count(); - std::cout << "It took me " << microsecondsUTC << " seconds." << std::endl; + std::cout << "It took me " << microsecondsUTC << " msec." << std::endl; } ~InflightMessageTtlManager() { timer_->disableTimer(); } void setDeadline(std::set&& ids) { auto expires_at = dispatcher_.timeSource().monotonicTime() + message_ack_timeout_; + std::cout << "set deadline: expires at "; + timeToString(expires_at); + for (const auto& id: ids) { + std::cout << id << std::endl; + } + std::cout << "==========" << std::endl; deadline_.emplace(expires_at, std::move(ids)); } - void received(uint32_t id) { received_ids_.emplace(id); } - private: Event::Dispatcher& dispatcher_; std::chrono::milliseconds message_ack_timeout_; Event::TimerPtr timer_; std::map, std::greater<>> deadline_; - std::set received_ids_; }; CriticalAccessLogger(const Grpc::RawAsyncClientSharedPtr& client, diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index 8c501f249dde0..a796c445f0539 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -342,7 +342,7 @@ TEST_P(CriticalAccessLogIntegrationTest, NoResponseFlow) { response_headers_bytes: 54 )EOF"))); - // test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_message_timeout", 1); + test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_message_timeout", 1); test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 1); cleanup(); } diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index fe0db0bcf7ee8..b5fa71a43a34a 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -173,8 +173,13 @@ AssertionResult TestUtility::waitForCounterEq(Stats::Store& store, const std::st std::chrono::milliseconds timeout, Event::Dispatcher* dispatcher) { Event::TestTimeSystem::RealTimeBound bound(timeout); - while (findCounter(store, name) == nullptr || findCounter(store, name)->value() != value) { + while (true) { + std::cout << "=== " << findCounter(store, name)->value() << std::endl; time_system.advanceTimeWait(std::chrono::milliseconds(10)); + std::cout << "=== " << findCounter(store, name)->value() << std::endl; + if (findCounter(store, name) != nullptr && findCounter(store, name)->value() == value) { + break; + } if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { std::string current_value; if (findCounter(store, name)) { From 1b9991d3e77248262e644ac3fbfd3a8c95e695f1 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 6 Sep 2021 20:33:43 +0900 Subject: [PATCH 24/39] test: fix corner-case for waiting stats change utility on integration test Signed-off-by: Shikugawa --- test/test_common/utility.cc | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index c2f68cc000c6d..23e31c52e70ba 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -173,8 +173,11 @@ AssertionResult TestUtility::waitForCounterEq(Stats::Store& store, const std::st std::chrono::milliseconds timeout, Event::Dispatcher* dispatcher) { Event::TestTimeSystem::RealTimeBound bound(timeout); - while (findCounter(store, name) == nullptr || findCounter(store, name)->value() != value) { + while (true) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); + if (findCounter(store, name) != nullptr && findCounter(store, name)->value() == value) { + break; + } if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { std::string current_value; if (findCounter(store, name)) { @@ -196,8 +199,11 @@ AssertionResult TestUtility::waitForCounterGe(Stats::Store& store, const std::st uint64_t value, Event::TestTimeSystem& time_system, std::chrono::milliseconds timeout) { Event::TestTimeSystem::RealTimeBound bound(timeout); - while (findCounter(store, name) == nullptr || findCounter(store, name)->value() < value) { + while (true) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); + if (findCounter(store, name) != nullptr && findCounter(store, name)->value() >= value) { + break; + } if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { return AssertionFailure() << fmt::format("timed out waiting for {} to be {}", name, value); } @@ -209,8 +215,11 @@ AssertionResult TestUtility::waitForGaugeGe(Stats::Store& store, const std::stri uint64_t value, Event::TestTimeSystem& time_system, std::chrono::milliseconds timeout) { Event::TestTimeSystem::RealTimeBound bound(timeout); - while (findGauge(store, name) == nullptr || findGauge(store, name)->value() < value) { + while (true) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); + if (findGauge(store, name) != nullptr && findGauge(store, name)->value() >= value) { + break; + } if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { return AssertionFailure() << fmt::format("timed out waiting for {} to be {}", name, value); } @@ -222,8 +231,11 @@ AssertionResult TestUtility::waitForGaugeEq(Stats::Store& store, const std::stri uint64_t value, Event::TestTimeSystem& time_system, std::chrono::milliseconds timeout) { Event::TestTimeSystem::RealTimeBound bound(timeout); - while (findGauge(store, name) == nullptr || findGauge(store, name)->value() != value) { + while (true) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); + if (findGauge(store, name) != nullptr && findGauge(store, name)->value() == value) { + break; + } if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { std::string current_value; if (findGauge(store, name)) { From 76ee0ca66ee67406d5573ae1b17d82dc1727311b Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Mon, 6 Sep 2021 23:42:28 +0900 Subject: [PATCH 25/39] fix Signed-off-by: Shikugawa --- .../common/grpc/buffered_async_client_impl.h | 32 +++++++++++-------- .../grpc/grpc_access_log_impl.cc | 4 +-- .../grpc/buffered_async_client_impl_test.cc | 4 +-- .../http_grpc_access_log_integration_test.cc | 3 +- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/source/common/grpc/buffered_async_client_impl.h b/source/common/grpc/buffered_async_client_impl.h index a4cecf5efb789..69d04910b2f3f 100644 --- a/source/common/grpc/buffered_async_client_impl.h +++ b/source/common/grpc/buffered_async_client_impl.h @@ -58,18 +58,7 @@ template class BufferedAsyncClient { return inflight_message_ids; } - void onSuccess(uint32_t message_id) { - if (message_buffer_.find(message_id) == message_buffer_.end()) { - return; - } - auto& buffer = message_buffer_.at(message_id); - - if (buffer.first == BufferState::Pending) { - const auto buffer_size = buffer.second.ByteSizeLong(); - current_buffer_bytes_ -= buffer_size; - message_buffer_.erase(message_id); - } - } + void onSuccess(uint32_t message_id) { eraseBuffer(message_id); } void onError(uint32_t message_id, bool rebuffer) { if (message_buffer_.find(message_id) == message_buffer_.end()) { @@ -79,7 +68,7 @@ template class BufferedAsyncClient { if (rebuffer) { message_buffer_.at(message_id).first = BufferState::Buffered; } else { - message_buffer_.erase(message_id); + eraseBuffer(message_id); } } @@ -91,9 +80,24 @@ template class BufferedAsyncClient { bool hasActiveStream() { return active_stream_ != nullptr; } - const absl::flat_hash_map>& messageBuffer() { return message_buffer_; } + const absl::flat_hash_map>& messageBuffer() { + return message_buffer_; + } private: + void eraseBuffer(uint32_t message_id) { + if (message_buffer_.find(message_id) == message_buffer_.end()) { + return; + } + auto& buffer = message_buffer_.at(message_id); + + if (buffer.first == BufferState::Pending) { + const auto buffer_size = buffer.second.ByteSizeLong(); + current_buffer_bytes_ -= buffer_size; + message_buffer_.erase(message_id); + } + } + uint32_t max_buffer_bytes_ = 0; const Protobuf::MethodDescriptor& service_method_; Grpc::AsyncStreamCallbacks& callbacks_; diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 0d32605be89f0..e7fbe43694ac3 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -113,8 +113,8 @@ CriticalAccessLogger::CriticalAccessLogger(const Grpc::RawAsyncClientSharedPtr& void CriticalAccessLogger::flush(CriticalAccessLogger::RequestType& message) { if (inflight_message_ttl_ == nullptr) { - inflight_message_ttl_ = - std::make_unique(dispatcher_, stats_, *client_, message_ack_timeout_); + inflight_message_ttl_ = std::make_unique( + dispatcher_, stats_, *client_, message_ack_timeout_); } const uint32_t message_id = client_->publishId(message); message.set_id(message_id); diff --git a/test/common/grpc/buffered_async_client_impl_test.cc b/test/common/grpc/buffered_async_client_impl_test.cc index 06ed75f20de6c..9833110a8e369 100644 --- a/test/common/grpc/buffered_async_client_impl_test.cc +++ b/test/common/grpc/buffered_async_client_impl_test.cc @@ -64,7 +64,7 @@ TEST_F(BufferedAsyncClientTest, BasicSendFlow) { EXPECT_EQ(1, buffered_client.sendBufferedMessages().size()); // Re-buffer, and transport. - buffered_client.bufferMessage(id); + buffered_client.onError(id, true); EXPECT_CALL(http_stream, sendData(_, _)).Times(2); EXPECT_CALL(http_stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); @@ -78,7 +78,7 @@ TEST_F(BufferedAsyncClientTest, BasicSendFlow) { // Clear existing messages. for (auto&& id : ids2) { - buffered_client.clearPendingMessage(id); + buffered_client.onSuccess(id); } // Successfully cleared pending messages. diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc index a796c445f0539..81a8744f99488 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc @@ -342,7 +342,8 @@ TEST_P(CriticalAccessLogIntegrationTest, NoResponseFlow) { response_headers_bytes: 54 )EOF"))); - test_server_->waitForCounterEq("access_logs.grpc_access_log.critical_logs_message_timeout", 1); + test_server_->notifyingStatsAllocator().waitForCounterFromStringEq( + "access_logs.grpc_access_log.critical_logs_message_timeout", 1); test_server_->waitForGaugeEq("access_logs.grpc_access_log.pending_critical_logs", 1); cleanup(); } From 9b88364ed422917bb1ffe082b26b436a2e7e49d2 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Tue, 7 Sep 2021 00:51:19 +0900 Subject: [PATCH 26/39] fix build Signed-off-by: Shikugawa --- source/extensions/access_loggers/grpc/grpc_access_log_impl.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 7a37325e93a2b..97226fc81d994 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -1,9 +1,5 @@ #pragma once -#include -#include -#include - #include #include From 8ada51f34e09495d6a70f1ed7ae782499cf5b7f1 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Tue, 7 Sep 2021 21:52:05 +0900 Subject: [PATCH 27/39] tidy Signed-off-by: Shikugawa --- test/common/grpc/buffered_async_client_impl_test.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/common/grpc/buffered_async_client_impl_test.cc b/test/common/grpc/buffered_async_client_impl_test.cc index 9833110a8e369..f2a2091ae10c3 100644 --- a/test/common/grpc/buffered_async_client_impl_test.cc +++ b/test/common/grpc/buffered_async_client_impl_test.cc @@ -15,8 +15,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::Eq; -using testing::Invoke; using testing::NiceMock; using testing::Return; using testing::ReturnRef; From a40041066ecd48d14a97db9f43a6555dc64f5c52 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 15 Sep 2021 07:13:19 +0000 Subject: [PATCH 28/39] grpc: implement BufferedAsyncClient for bidi gRPC stream Signed-off-by: Shikugawa --- source/common/grpc/BUILD | 9 ++ source/common/grpc/buffered_async_client.h | 114 ++++++++++++++++++ test/common/grpc/BUILD | 15 +++ .../common/grpc/buffered_async_client_test.cc | 113 +++++++++++++++++ 4 files changed, 251 insertions(+) create mode 100644 source/common/grpc/buffered_async_client.h create mode 100644 test/common/grpc/buffered_async_client_test.cc diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index 8e3cc89fd7e77..6729a434b2466 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -206,3 +206,12 @@ envoy_cc_library( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) + +envoy_cc_library( + name = "buffered_async_client_lib", + hdrs = ["buffered_async_client.h"], + deps = [ + ":typed_async_client_lib", + "//source/common/protobuf:utility_lib", + ], +) diff --git a/source/common/grpc/buffered_async_client.h b/source/common/grpc/buffered_async_client.h new file mode 100644 index 0000000000000..e2dca0a90c3e2 --- /dev/null +++ b/source/common/grpc/buffered_async_client.h @@ -0,0 +1,114 @@ +#pragma once + +#include "source/common/grpc/typed_async_client.h" +#include "source/common/protobuf/utility.h" + +namespace Envoy { +namespace Grpc { + +enum class BufferState { Buffered, PendingFlush }; + +template class BufferedAsyncClient { +public: + BufferedAsyncClient(uint32_t max_buffer_bytes, const Protobuf::MethodDescriptor& service_method, + Grpc::AsyncStreamCallbacks& callbacks, + const Grpc::AsyncClient& client) + : max_buffer_bytes_(max_buffer_bytes), service_method_(service_method), callbacks_(callbacks), + client_(client) {} + + virtual ~BufferedAsyncClient() { cleanup(); } + + uint32_t publishId(RequestType& message) { return MessageUtil::hash(message); } + + void bufferMessage(uint32_t id, RequestType& message) { + const auto buffer_size = message.ByteSizeLong(); + if (current_buffer_bytes_ + buffer_size > max_buffer_bytes_) { + return; + } + + message_buffer_[id] = std::make_pair(BufferState::Buffered, message); + current_buffer_bytes_ += buffer_size; + } + + absl::flat_hash_set sendBufferedMessages() { + if (active_stream_ == nullptr) { + active_stream_ = + client_.start(service_method_, callbacks_, Http::AsyncClient::StreamOptions()); + } + + if (active_stream_->isAboveWriteBufferHighWatermark()) { + return {}; + } + + absl::flat_hash_set inflight_message_ids; + + for (auto&& it : message_buffer_) { + const auto id = it.first; + auto& state = it.second.first; + auto& message = it.second.second; + + if (state == BufferState::PendingFlush) { + continue; + } + + state = BufferState::PendingFlush; + inflight_message_ids.emplace(id); + active_stream_->sendMessage(message, false); + } + + return inflight_message_ids; + } + + void onSuccess(uint32_t message_id) { erasePendingMessage(message_id); } + + void onError(uint32_t message_id) { + if (message_buffer_.find(message_id) == message_buffer_.end()) { + return; + } + message_buffer_.at(message_id).first = BufferState::Buffered; + } + + void cleanup() { + if (active_stream_ != nullptr) { + active_stream_ = nullptr; + } + } + + bool hasActiveStream() { return active_stream_ != nullptr; } + + const absl::flat_hash_map>& messageBuffer() { + return message_buffer_; + } + +private: + void erasePendingMessage(uint32_t message_id) { + if (message_buffer_.find(message_id) == message_buffer_.end()) { + return; + } + auto& buffer = message_buffer_.at(message_id); + + // There may be cases where the buffer status is not PendingFlush when + // this function is called. For example, a message_buffer that was + // PendingFlush may become Buffered due to an external state change + // (e.g. re-buffering due to timeout). + if (buffer.first == BufferState::PendingFlush) { + const auto buffer_size = buffer.second.ByteSizeLong(); + current_buffer_bytes_ -= buffer_size; + message_buffer_.erase(message_id); + } + } + + uint32_t max_buffer_bytes_ = 0; + const Protobuf::MethodDescriptor& service_method_; + Grpc::AsyncStreamCallbacks& callbacks_; + Grpc::AsyncClient client_; + Grpc::AsyncStream active_stream_; + absl::flat_hash_map> message_buffer_; + uint32_t current_buffer_bytes_ = 0; +}; + +template +using BufferedAsyncClientPtr = std::unique_ptr>; + +} // namespace Grpc +} // namespace Envoy diff --git a/test/common/grpc/BUILD b/test/common/grpc/BUILD index c4e37301fa708..ca9a7a068b666 100644 --- a/test/common/grpc/BUILD +++ b/test/common/grpc/BUILD @@ -186,3 +186,18 @@ envoy_cc_test_library( "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) + +envoy_cc_test( + name = "buffered_async_client_test", + srcs = ["buffered_async_client_test.cc"], + deps = [ + "//source/common/grpc:async_client_lib", + "//source/common/grpc:buffered_async_client_lib", + "//test/mocks/http:http_mocks", + "//test/mocks/tracing:tracing_mocks", + "//test/mocks/upstream:cluster_manager_mocks", + "//test/proto:helloworld_proto_cc_proto", + "//test/test_common:test_time_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) diff --git a/test/common/grpc/buffered_async_client_test.cc b/test/common/grpc/buffered_async_client_test.cc new file mode 100644 index 0000000000000..4caec0900132f --- /dev/null +++ b/test/common/grpc/buffered_async_client_test.cc @@ -0,0 +1,113 @@ +#include "envoy/config/core/v3/grpc_service.pb.h" + +#include "source/common/grpc/async_client_impl.h" +#include "source/common/grpc/buffered_async_client.h" +#include "source/common/network/address_impl.h" +#include "source/common/network/socket_impl.h" + +#include "test/mocks/http/mocks.h" +#include "test/mocks/tracing/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" +#include "test/proto/helloworld.pb.h" +#include "test/test_common/test_time.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Grpc { +namespace { + +class BufferedAsyncClientTest : public testing::Test { +public: + BufferedAsyncClientTest() + : method_descriptor_(helloworld::Greeter::descriptor()->FindMethodByName("SayHello")) { + config_.mutable_envoy_grpc()->set_cluster_name("test_cluster"); + + cm_.initializeThreadLocalClusters({"test_cluster"}); + ON_CALL(cm_.thread_local_cluster_, httpAsyncClient()).WillByDefault(ReturnRef(http_client_)); + } + + const Protobuf::MethodDescriptor* method_descriptor_; + envoy::config::core::v3::GrpcService config_; + NiceMock cm_; + NiceMock http_client_; +}; + +TEST_F(BufferedAsyncClientTest, BasicSendFlow) { + Http::MockAsyncClientStream http_stream; + EXPECT_CALL(http_client_, start(_, _)).WillOnce(Return(&http_stream)); + EXPECT_CALL(http_stream, sendHeaders(_, _)); + EXPECT_CALL(http_stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + EXPECT_CALL(http_stream, sendData(_, _)); + EXPECT_CALL(http_stream, reset()); + + DangerousDeprecatedTestTime test_time_; + auto raw_client = std::make_shared(cm_, config_, test_time_.timeSystem()); + AsyncClient client(raw_client); + + NiceMock> callback; + BufferedAsyncClient buffered_client( + 100000, *method_descriptor_, callback, client); + + helloworld::HelloRequest request; + request.set_name("Alice"); + auto id = buffered_client.publishId(request); + buffered_client.bufferMessage(id, request); + EXPECT_EQ(1, buffered_client.sendBufferedMessages().size()); + + // Re-buffer, and transport. + buffered_client.onError(id); + + EXPECT_CALL(http_stream, sendData(_, _)).Times(2); + EXPECT_CALL(http_stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + + helloworld::HelloRequest request2; + request2.set_name("Bob"); + auto id2 = buffered_client.publishId(request2); + buffered_client.bufferMessage(id2, request2); + auto ids2 = buffered_client.sendBufferedMessages(); + EXPECT_EQ(2, ids2.size()); + + // Clear existing messages. + for (auto&& id : ids2) { + buffered_client.onSuccess(id); + } + + // Successfully cleared pending messages. + EXPECT_CALL(http_stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + auto ids3 = buffered_client.sendBufferedMessages(); + EXPECT_EQ(0, ids3.size()); +} + +TEST_F(BufferedAsyncClientTest, BufferLimitExceeded) { + Http::MockAsyncClientStream http_stream; + EXPECT_CALL(http_client_, start(_, _)).WillOnce(Return(&http_stream)); + EXPECT_CALL(http_stream, sendHeaders(_, _)); + EXPECT_CALL(http_stream, isAboveWriteBufferHighWatermark()).WillOnce(Return(false)); + EXPECT_CALL(http_stream, reset()); + + DangerousDeprecatedTestTime test_time_; + auto raw_client = std::make_shared(cm_, config_, test_time_.timeSystem()); + AsyncClient client(raw_client); + + NiceMock> callback; + BufferedAsyncClient buffered_client( + 0, *method_descriptor_, callback, client); + + helloworld::HelloRequest request; + request.set_name("Alice"); + auto id = buffered_client.publishId(request); + buffered_client.bufferMessage(id, request); + + EXPECT_EQ(0, buffered_client.sendBufferedMessages().size()); +} + +} // namespace +} // namespace Grpc +} // namespace Envoy From 7cfdf0ffbe25a4431eb6bdd0d03b55267ed8c1c2 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 15 Sep 2021 10:51:54 +0000 Subject: [PATCH 29/39] fix Signed-off-by: Shikugawa --- .../common/grpc_access_logger.h | 67 ++++++++--------- .../grpc/grpc_access_log_impl.cc | 71 +++++++++---------- .../grpc/grpc_access_log_impl.h | 66 ++++++++++++----- .../open_telemetry/grpc_access_log_impl.cc | 18 ++--- .../open_telemetry/grpc_access_log_impl.h | 5 +- .../grpc/grpc_access_log_impl_test.cc | 5 +- .../grpc/http_grpc_access_log_impl_test.cc | 3 + .../open_telemetry/access_log_impl_test.cc | 3 + .../grpc_access_log_impl_test.cc | 7 +- 9 files changed, 140 insertions(+), 105 deletions(-) diff --git a/source/extensions/access_loggers/common/grpc_access_logger.h b/source/extensions/access_loggers/common/grpc_access_logger.h index 4691f267803ee..3824611911a8d 100644 --- a/source/extensions/access_loggers/common/grpc_access_logger.h +++ b/source/extensions/access_loggers/common/grpc_access_logger.h @@ -43,6 +43,15 @@ template class GrpcAccessLogger { virtual ~GrpcAccessLogger() = default; + /** + * Start interval log flusher. + * @param dispatcher Event dispatcher to create timer to flush log message. + * @param buffer_flush_interval_msec Buffer flush interval msec. + */ + virtual void + startIntervalFlushTimer(Event::Dispatcher& dispatcher, + const std::chrono::milliseconds buffer_flush_interval_msec) PURE; + /** * Log http access entry. * @param entry supplies the access log to send. @@ -161,28 +170,23 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger; - GrpcAccessLogger(const Grpc::RawAsyncClientSharedPtr& client, - std::chrono::milliseconds buffer_flush_interval_msec, - uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, + GrpcAccessLogger(const Grpc::RawAsyncClientSharedPtr& client, uint64_t max_buffer_size_bytes, Stats::Scope& scope, std::string access_log_prefix, const Protobuf::MethodDescriptor& service_method) - : client_(client, service_method), buffer_flush_interval_msec_(buffer_flush_interval_msec), - flush_timer_(dispatcher.createTimer([this]() { - flush(); - flushCriticalMessage(); - flush_timer_->enableTimer(buffer_flush_interval_msec_); - })), - max_buffer_size_bytes_(max_buffer_size_bytes), - stats_({ALL_GRPC_ACCESS_LOGGER_STATS(POOL_COUNTER_PREFIX(scope, access_log_prefix))}) { - flush_timer_->enableTimer(buffer_flush_interval_msec_); - } + : client_(client, service_method), max_buffer_size_bytes_(max_buffer_size_bytes), + stats_({ALL_GRPC_ACCESS_LOGGER_STATS(POOL_COUNTER_PREFIX(scope, access_log_prefix))}) {} - void log(HttpLogProto&& entry, bool is_critical) { - if (is_critical) { - logCritical(std::move(entry)); - return; - } + void + startIntervalFlushTimer(Event::Dispatcher& dispatcher, + const std::chrono::milliseconds buffer_flush_interval_msec) override { + flush_timer_ = dispatcher.createTimer([this, buffer_flush_interval_msec]() { + flush(); + flush_timer_->enableTimer(buffer_flush_interval_msec); + }); + flush_timer_->enableTimer(buffer_flush_interval_msec); + } + void log(HttpLogProto&& entry, bool) override { if (!canLogMore()) { return; } @@ -193,7 +197,7 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger= max_buffer_size_bytes_) { @@ -202,18 +206,6 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger client_; - LogRequest message_; - -private: - virtual bool isEmpty() PURE; - virtual void initMessage() PURE; - virtual void addEntry(HttpLogProto&& entry) PURE; - virtual void addEntry(TcpLogProto&& entry) PURE; - virtual void clearMessage() { message_.Clear(); } - virtual void flushCriticalMessage() {} - virtual void logCritical(HttpLogProto&&) {} - void flush() { if (isEmpty()) { // Nothing to flush. @@ -231,6 +223,17 @@ class GrpcAccessLogger : public Detail::GrpcAccessLogger client_; + LogRequest message_; + Event::TimerPtr flush_timer_; + +private: + virtual bool isEmpty() PURE; + virtual void initMessage() PURE; + virtual void addEntry(HttpLogProto&& entry) PURE; + virtual void addEntry(TcpLogProto&& entry) PURE; + virtual void clearMessage() { message_.Clear(); } + bool canLogMore() { if (max_buffer_size_bytes_ == 0 || approximate_message_size_bytes_ < max_buffer_size_bytes_) { stats_.logs_written_.inc(); @@ -245,8 +248,6 @@ class GrpcAccessLogger : public Detail::GrpcAccessLoggerFindMethodByName( "envoy.service.accesslog.v3.AccessLogService.StreamAccessLogs")), max_critical_message_size_bytes_(max_buffer_size_bytes), log_name_(config.log_name()), local_info_(local_info) { - critical_logger_ = std::make_unique( + critical_log_client_ = std::make_unique( client, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v3.AccessLogService.CriticalAccessLogs"), - dispatcher, scope, local_info_, log_name_, PROTOBUF_GET_MS_OR_DEFAULT(config, message_ack_timeout, 5000), + dispatcher, scope, local_info, log_name_, + PROTOBUF_GET_MS_OR_DEFAULT(config, message_ack_timeout, 5000), PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_pending_buffer_size_bytes, 16384)); } @@ -58,6 +59,12 @@ bool GrpcAccessLoggerImpl::isEmpty() { return !message_.has_http_logs() && !message_.has_tcp_logs(); } +void GrpcAccessLoggerImpl::initMessage() { + auto* identifier = message_.mutable_identifier(); + *identifier->mutable_node() = local_info_.node(); + identifier->set_log_name(log_name_); +} + bool GrpcAccessLoggerImpl::isCriticalMessageEmpty() { return !critical_message_.message().has_http_logs() && !critical_message_.message().has_tcp_logs(); @@ -69,53 +76,32 @@ void GrpcAccessLoggerImpl::flushCriticalMessage() { } approximate_critical_message_size_bytes_ = 0; - critical_logger_->flush(critical_message_); + critical_log_client_->flush(critical_message_); clearCriticalMessage(); } -void GrpcAccessLoggerImpl::logCritical(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry) { - approximate_critical_message_size_bytes_ += entry.ByteSizeLong(); - addCriticalMessageEntry(std::move(entry)); - - if (approximate_critical_message_size_bytes_ >= max_critical_message_size_bytes_) { - flushCriticalMessage(); - } -} - -void GrpcAccessLoggerImpl::initMessage() { - auto* identifier = message_.mutable_identifier(); - *identifier->mutable_node() = local_info_.node(); - identifier->set_log_name(log_name_); -} - -CriticalAccessLogger::CriticalAccessLogger(const Grpc::RawAsyncClientSharedPtr& client, - const Protobuf::MethodDescriptor& method, - Event::Dispatcher& dispatcher, Stats::Scope& scope, - const LocalInfo::LocalInfo& local_info, - const std::string& log_name, - uint64_t message_ack_timeout, - uint64_t max_pending_buffer_size_bytes) +GrpcCriticalAccessLogClient::GrpcCriticalAccessLogClient( + const Grpc::RawAsyncClientSharedPtr& client, const Protobuf::MethodDescriptor& method, + Event::Dispatcher& dispatcher, Stats::Scope& scope, const LocalInfo::LocalInfo& local_info, + const std::string& log_name, uint64_t message_ack_timeout, + uint64_t max_pending_buffer_size_bytes) : dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout), stats_({CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS( POOL_COUNTER_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()), POOL_GAUGE_PREFIX(scope, GRPC_LOG_STATS_PREFIX.data()))}), - local_info_(local_info), - log_name_(log_name), - stream_callback_(*this) { + local_info_(local_info), log_name_(log_name), stream_callback_(*this) { client_ = std::make_unique>( max_pending_buffer_size_bytes, method, stream_callback_, client); } -void CriticalAccessLogger::flush(CriticalAccessLogger::RequestType& message) { +void GrpcCriticalAccessLogClient::flush(GrpcCriticalAccessLogClient::RequestType& message) { if (inflight_message_ttl_ == nullptr) { inflight_message_ttl_ = std::make_unique( dispatcher_, stats_, *client_, message_ack_timeout_); } if (!client_->hasActiveStream()) { - auto* identifier = message.mutable_message()->mutable_identifier(); - *identifier->mutable_node() = local_info_.node(); - identifier->set_log_name(log_name_); + setLogIdentifier(message); } const uint32_t message_id = client_->publishId(message); @@ -125,6 +111,12 @@ void CriticalAccessLogger::flush(CriticalAccessLogger::RequestType& message) { inflight_message_ttl_->setDeadline(client_->sendBufferedMessages()); } +void GrpcCriticalAccessLogClient::setLogIdentifier(RequestType& message) { + auto* identifier = message.mutable_message()->mutable_identifier(); + *identifier->mutable_node() = local_info_.node(); + identifier->set_log_name(log_name_); +} + GrpcAccessLoggerCacheImpl::GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& async_client_manager, Stats::Scope& scope, ThreadLocal::SlotAllocator& tls, @@ -136,9 +128,10 @@ GrpcAccessLoggerImpl::SharedPtr GrpcAccessLoggerCacheImpl::createLogger( const Grpc::RawAsyncClientSharedPtr& client, std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, Stats::Scope& scope) { - return std::make_shared(client, config, buffer_flush_interval_msec, - max_buffer_size_bytes, dispatcher, local_info_, - scope); + auto logger = std::make_shared(client, config, max_buffer_size_bytes, + dispatcher, local_info_, scope); + logger->startIntervalFlushTimer(dispatcher, buffer_flush_interval_msec); + return logger; } } // namespace GrpcCommon diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 0ba456b4976c6..871502cea34c2 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -30,17 +30,17 @@ static constexpr absl::string_view GRPC_LOG_STATS_PREFIX = "access_logs.grpc_acc COUNTER(critical_logs_ack_received) \ GAUGE(pending_critical_logs, Accumulate) -struct CriticalAccessLoggerGrpcClientStats { +struct GrpcCriticalAccessLogClientGrpcClientStats { CRITICAL_ACCESS_LOGGER_GRPC_CLIENT_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) }; -class CriticalAccessLogger { +class GrpcCriticalAccessLogClient { public: using RequestType = envoy::service::accesslog::v3::CriticalAccessLogsMessage; using ResponseType = envoy::service::accesslog::v3::CriticalAccessLogsResponse; struct CriticalLogStream : public Grpc::AsyncStreamCallbacks { - explicit CriticalLogStream(CriticalAccessLogger& parent) : parent_(parent) {} + explicit CriticalLogStream(GrpcCriticalAccessLogClient& parent) : parent_(parent) {} // Grpc::AsyncStreamCallbacks void onCreateInitialMetadata(Http::RequestHeaderMap&) override {} @@ -67,13 +67,13 @@ class CriticalAccessLogger { parent_.client_->cleanup(); } - CriticalAccessLogger& parent_; + GrpcCriticalAccessLogClient& parent_; }; class InflightMessageTtlManager { public: InflightMessageTtlManager(Event::Dispatcher& dispatcher, - CriticalAccessLoggerGrpcClientStats& stats, + GrpcCriticalAccessLogClientGrpcClientStats& stats, Grpc::BufferedAsyncClient& client, std::chrono::milliseconds message_ack_timeout) : dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout) { @@ -129,20 +129,22 @@ class CriticalAccessLogger { std::map, std::greater<>> deadline_; }; - CriticalAccessLogger(const Grpc::RawAsyncClientSharedPtr& client, - const Protobuf::MethodDescriptor& method, Event::Dispatcher& dispatcher, - Stats::Scope& scope, const LocalInfo::LocalInfo& local_info, const std::string& log_name, - uint64_t message_ack_timeout, - uint64_t max_pending_buffer_size_bytes); + GrpcCriticalAccessLogClient(const Grpc::RawAsyncClientSharedPtr& client, + const Protobuf::MethodDescriptor& method, + Event::Dispatcher& dispatcher, Stats::Scope& scope, + const LocalInfo::LocalInfo& local_info, const std::string& log_name, + uint64_t message_ack_timeout, uint64_t max_pending_buffer_size_bytes); void flush(RequestType& message); private: friend CriticalLogStream; + void setLogIdentifier(RequestType& request); + Event::Dispatcher& dispatcher_; std::chrono::milliseconds message_ack_timeout_; - CriticalAccessLoggerGrpcClientStats stats_; + GrpcCriticalAccessLogClientGrpcClientStats stats_; const LocalInfo::LocalInfo& local_info_; const std::string log_name_; CriticalLogStream stream_callback_; @@ -156,16 +158,50 @@ class GrpcAccessLoggerImpl envoy::service::accesslog::v3::StreamAccessLogsMessage, envoy::service::accesslog::v3::StreamAccessLogsResponse> { public: + using TcpLogProto = envoy::data::accesslog::v3::TCPAccessLogEntry; + using HttpLogProto = envoy::data::accesslog::v3::HTTPAccessLogEntry; + using BaseLogger = + Common::GrpcAccessLogger; + GrpcAccessLoggerImpl( const Grpc::RawAsyncClientSharedPtr& client, const envoy::extensions::access_loggers::grpc::v3::CommonGrpcAccessLogConfig& config, - std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, - Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope); + uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, + const LocalInfo::LocalInfo& local_info, Stats::Scope& scope); + + void + startIntervalFlushTimer(Event::Dispatcher& dispatcher, + const std::chrono::milliseconds buffer_flush_interval_msec) override { + flush_timer_ = dispatcher.createTimer([this, buffer_flush_interval_msec]() { + flush(); + flushCriticalMessage(); + flush_timer_->enableTimer(buffer_flush_interval_msec); + }); + flush_timer_->enableTimer(buffer_flush_interval_msec); + } + + void log(HttpLogProto&& entry, bool is_critical) override { + if (is_critical) { + approximate_critical_message_size_bytes_ += entry.ByteSizeLong(); + addCriticalMessageEntry(std::move(entry)); + + if (approximate_critical_message_size_bytes_ >= max_critical_message_size_bytes_) { + flushCriticalMessage(); + } + return; + } + BaseLogger::log(std::move(entry), false); + } + + void log(TcpLogProto&& entry, bool) override { BaseLogger::log(std::move(entry), false); } private: bool isCriticalMessageEmpty(); void addCriticalMessageEntry(envoy::data::accesslog::v3::HTTPAccessLogEntry&& entry); void addCriticalMessageEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry); + void flushCriticalMessage(); void clearCriticalMessage() { critical_message_.Clear(); } // Extensions::AccessLoggers::GrpcCommon::GrpcAccessLogger @@ -173,12 +209,10 @@ class GrpcAccessLoggerImpl void addEntry(envoy::data::accesslog::v3::TCPAccessLogEntry&& entry) override; bool isEmpty() override; void initMessage() override; - void flushCriticalMessage() override; - void logCritical(envoy::data::accesslog::v3::HTTPAccessLogEntry&&) override; uint64_t approximate_critical_message_size_bytes_ = 0; uint64_t max_critical_message_size_bytes_ = 0; - std::unique_ptr critical_logger_; + std::unique_ptr critical_log_client_; envoy::service::accesslog::v3::CriticalAccessLogsMessage critical_message_; const std::string log_name_; const LocalInfo::LocalInfo& local_info_; diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc index 215f7cfba9e4e..814f1f4b0431c 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.cc @@ -20,12 +20,11 @@ namespace Extensions { namespace AccessLoggers { namespace OpenTelemetry { -GrpcAccessLoggerImpl::GrpcAccessLoggerImpl( - const Grpc::RawAsyncClientSharedPtr& client, std::string log_name, - std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, - Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope) - : GrpcAccessLogger(client, buffer_flush_interval_msec, max_buffer_size_bytes, dispatcher, scope, - GRPC_LOG_STATS_PREFIX, +GrpcAccessLoggerImpl::GrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, + std::string log_name, uint64_t max_buffer_size_bytes, + const LocalInfo::LocalInfo& local_info, + Stats::Scope& scope) + : GrpcAccessLogger(client, max_buffer_size_bytes, scope, GRPC_LOG_STATS_PREFIX, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "opentelemetry.proto.collector.logs.v1.LogsService.Export")) { initMessageRoot(log_name, local_info); @@ -77,9 +76,10 @@ GrpcAccessLoggerImpl::SharedPtr GrpcAccessLoggerCacheImpl::createLogger( const Grpc::RawAsyncClientSharedPtr& client, std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, Stats::Scope& scope) { - return std::make_shared(client, config.log_name(), - buffer_flush_interval_msec, max_buffer_size_bytes, - dispatcher, local_info_, scope); + auto logger = std::make_shared(client, config.log_name(), + max_buffer_size_bytes, local_info_, scope); + logger->startIntervalFlushTimer(dispatcher, buffer_flush_interval_msec); + return logger; } } // namespace OpenTelemetry diff --git a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h index 7af83f529de4c..263e541c8ba4a 100644 --- a/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/open_telemetry/grpc_access_log_impl.h @@ -36,9 +36,8 @@ class GrpcAccessLoggerImpl opentelemetry::proto::collector::logs::v1::ExportLogsServiceResponse> { public: GrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, std::string log_name, - std::chrono::milliseconds buffer_flush_interval_msec, - uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, - const LocalInfo::LocalInfo& local_info, Stats::Scope& scope); + uint64_t max_buffer_size_bytes, const LocalInfo::LocalInfo& local_info, + Stats::Scope& scope); private: void initMessageRoot(const std::string& log_name, const LocalInfo::LocalInfo& local_info); diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 1c48bef05118f..d3a06e0ccdebe 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -91,8 +91,9 @@ class GrpcAccessLoggerImplTest : public testing::Test { EXPECT_CALL(*timer_, enableTimer(_, _)); config_.set_log_name("test_log_name"); logger_ = std::make_unique(Grpc::RawAsyncClientPtr{async_client_}, - config_, FlushInterval, BUFFER_SIZE_BYTES, - dispatcher_, local_info_, stats_store_); + config_, BUFFER_SIZE_BYTES, dispatcher_, + local_info_, stats_store_); + logger_->startIntervalFlushTimer(dispatcher_, FlushInterval); } Grpc::MockAsyncClient* async_client_; diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index b01ed049c3b73..b7abb7619a315 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -39,6 +39,9 @@ using envoy::data::accesslog::v3::HTTPAccessLogEntry; class MockGrpcAccessLogger : public GrpcCommon::GrpcAccessLogger { public: // GrpcAccessLogger + MOCK_METHOD(void, startIntervalFlushTimer, + (Event::Dispatcher & dispatcher, + const std::chrono::milliseconds buffer_flush_interval_msec)); MOCK_METHOD(void, log, (HTTPAccessLogEntry && entry, bool is_critical)); MOCK_METHOD(void, log, (envoy::data::accesslog::v3::TCPAccessLogEntry && entry, bool is_critical)); diff --git a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc index 00f903fbbb68d..bc6788efcf465 100644 --- a/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/access_log_impl_test.cc @@ -44,6 +44,9 @@ namespace { class MockGrpcAccessLogger : public GrpcAccessLogger { public: // GrpcAccessLogger + MOCK_METHOD(void, startIntervalFlushTimer, + (Event::Dispatcher & dispatcher, + std::chrono::milliseconds buffer_flush_interval_msec)); MOCK_METHOD(void, log, (LogRecord && entry, bool)); MOCK_METHOD(void, log, (ProtobufWkt::Empty && entry, bool)); }; diff --git a/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc index 53e206663737a..feed201c976e9 100644 --- a/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/open_telemetry/grpc_access_log_impl_test.cc @@ -79,9 +79,10 @@ class GrpcAccessLoggerImplTest : public testing::Test { : async_client_(new Grpc::MockAsyncClient), timer_(new Event::MockTimer(&dispatcher_)), grpc_access_logger_impl_test_helper_(local_info_, async_client_) { EXPECT_CALL(*timer_, enableTimer(_, _)); - logger_ = std::make_unique( - Grpc::RawAsyncClientPtr{async_client_}, "test_log_name", FlushInterval, BUFFER_SIZE_BYTES, - dispatcher_, local_info_, stats_store_); + logger_ = std::make_unique(Grpc::RawAsyncClientPtr{async_client_}, + "test_log_name", BUFFER_SIZE_BYTES, + local_info_, stats_store_); + logger_->startIntervalFlushTimer(dispatcher_, FlushInterval); } Grpc::MockAsyncClient* async_client_; From 0793debcb2f3550f9dbf5301cf2a68043c5b5bd7 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 15 Sep 2021 10:54:00 +0000 Subject: [PATCH 30/39] fix Signed-off-by: Shikugawa --- source/extensions/access_loggers/grpc/http_config.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/extensions/access_loggers/grpc/http_config.cc b/source/extensions/access_loggers/grpc/http_config.cc index 3a289479666c8..da8f22fc87680 100644 --- a/source/extensions/access_loggers/grpc/http_config.cc +++ b/source/extensions/access_loggers/grpc/http_config.cc @@ -31,7 +31,9 @@ AccessLog::InstanceSharedPtr HttpGrpcAccessLogFactory::createAccessLogInstance( if (service_config.has_envoy_grpc()) { context.clusterManager().checkActiveStaticCluster(service_config.envoy_grpc().cluster_name()); } - return std::make_shared(std::move(filter), proto_config, GrpcCommon::getGrpcAccessLoggerCacheSingleton(context), context); + return std::make_shared(std::move(filter), proto_config, + GrpcCommon::getGrpcAccessLoggerCacheSingleton(context), + context); } ProtobufTypes::MessagePtr HttpGrpcAccessLogFactory::createEmptyConfigProto() { From 599b05a230b5ceee5c39bacb5088a2adeebf6b8d Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 15 Sep 2021 10:56:33 +0000 Subject: [PATCH 31/39] fix Signed-off-by: Shikugawa --- api/envoy/extensions/access_loggers/grpc/v3/als.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/envoy/extensions/access_loggers/grpc/v3/als.proto b/api/envoy/extensions/access_loggers/grpc/v3/als.proto index 29904142368e1..6b37370f80a89 100644 --- a/api/envoy/extensions/access_loggers/grpc/v3/als.proto +++ b/api/envoy/extensions/access_loggers/grpc/v3/als.proto @@ -89,8 +89,8 @@ message CommonGrpcAccessLogConfig { repeated string filter_state_objects_to_log = 5; // Define the log condition for critical access logs. - // Logs that match the filter are not forwarded to the normal endpoint, - // but are sent to a special endpoint. + // Logs that match the filter are not sent to `StreamAccessLogs`, + // but are sent to `CriticalAccessLogs`. config.accesslog.v3.AccessLogFilter critical_buffer_log_filter = 7; // The time to wait for an ACK message. If no ACK message is returned after this time, From 6c43c6c8b5ff2abcce8c5b2bdbb0e4392a5e310f Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 15 Sep 2021 13:34:34 +0000 Subject: [PATCH 32/39] fix Signed-off-by: Shikugawa --- .../grpc/grpc_access_log_impl.h | 122 ++++++++++++------ 1 file changed, 86 insertions(+), 36 deletions(-) diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 871502cea34c2..347775354098d 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -76,55 +76,105 @@ class GrpcCriticalAccessLogClient { GrpcCriticalAccessLogClientGrpcClientStats& stats, Grpc::BufferedAsyncClient& client, std::chrono::milliseconds message_ack_timeout) - : dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout) { - timer_ = dispatcher_.createTimer([this, &client, &stats] { - const auto now = dispatcher_.timeSource().monotonicTime(); - std::vector expired_timepoints; - absl::flat_hash_set expired_message_ids; - - auto it = deadline_.lower_bound(now); - while (it != deadline_.end()) { - for (auto&& id : it->second) { - expired_message_ids.emplace(id); - } - expired_timepoints.push_back(it->first); - ++it; - } + : dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout), stats_(stats), + client_(client) {} - for (auto&& id : expired_message_ids) { - const auto& message_buffer = client.messageBuffer(); + ~InflightMessageTtlManager() { + if (timer_ != nullptr) { + timer_->disableTimer(); + } + } - if (message_buffer.find(id) == message_buffer.end()) { - continue; - } + void setDeadline(absl::flat_hash_set&& ids) { + auto expires_at = dispatcher_.timeSource().monotonicTime() + message_ack_timeout_; + deadline_.emplace(expires_at, std::move(ids)); - auto& message = message_buffer.at(id); - if (message.first == Grpc::BufferState::PendingFlush) { - client.onError(id); - stats.critical_logs_message_timeout_.inc(); - } - } + if (timer_ == nullptr) { + timer_ = dispatcher_.createTimer([this] { + callback(); + restartTimer(); + }); - for (auto&& timepoint : expired_timepoints) { - deadline_.erase(timepoint); - } + // Since this if section is never called except at the first startup, + // it is sufficient to set message_ack_timeout. + setTimer(message_ack_timeout_); + } + } - timer_->enableTimer(message_ack_timeout_); - }); + private: + void restartTimer() { + if (timer_ != nullptr) { + return; + } - timer_->enableTimer(message_ack_timeout_); + if (deadline_.empty()) { + timer_ = nullptr; + return; + } + + // When restarting the timer, set the earliest time point among the currently remaining + // messages. This will allow you to enforce an accurate timeout. + const auto earlier_timepoint = deadline_.rbegin()->first; + auto timer_duration = std::chrono::duration_cast( + earlier_timepoint - dispatcher_.timeSource().monotonicTime()); + + // A corner case may occur where the timer is not subject to a timeout + // at the stage where it fires, but becomes subject to a timeout at the + // stage where the timer glue start takes place. This if statement is + // a mechanism to prevent this from happening. + setTimer(timer_duration.count() > 0 ? timer_duration : message_ack_timeout_); } - ~InflightMessageTtlManager() { timer_->disableTimer(); } + void setTimer(std::chrono::milliseconds duration) { + timer_ = dispatcher_.createTimer([this] { + callback(); + restartTimer(); + }); + timer_->enableTimer(duration); + } - void setDeadline(absl::flat_hash_set&& ids) { - auto expires_at = dispatcher_.timeSource().monotonicTime() + message_ack_timeout_; - deadline_.emplace(expires_at, std::move(ids)); + void callback() { + const auto now = dispatcher_.timeSource().monotonicTime(); + std::vector expired_timepoints; + absl::flat_hash_set expired_message_ids; + + // Extract timeout message ids. + auto it = deadline_.lower_bound(now); + while (it != deadline_.end()) { + for (auto&& id : it->second) { + expired_message_ids.emplace(id); + } + expired_timepoints.push_back(it->first); + ++it; + } + + // Clear buffered message ids on the set of waiting timeout. + for (auto&& timepoint : expired_timepoints) { + deadline_.erase(timepoint); + } + + // Restore pending messages to buffer due to timeout. + for (auto&& id : expired_message_ids) { + const auto& message_buffer = client_.messageBuffer(); + + if (message_buffer.find(id) == message_buffer.end()) { + continue; + } + + auto& message = message_buffer.at(id); + if (message.first == Grpc::BufferState::PendingFlush) { + client_.onError(id); + stats_.critical_logs_message_timeout_.inc(); + } + } + + timer_ = nullptr; } - private: Event::Dispatcher& dispatcher_; std::chrono::milliseconds message_ack_timeout_; + GrpcCriticalAccessLogClientGrpcClientStats& stats_; + Grpc::BufferedAsyncClient& client_; Event::TimerPtr timer_; std::map, std::greater<>> deadline_; }; From 44dd7efaac6c2f5b0353b194703b7661caf18062 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 15 Sep 2021 15:15:52 +0000 Subject: [PATCH 33/39] fix Signed-off-by: Shikugawa --- .../extensions/access_loggers/grpc/v3/BUILD | 4 +- .../grpc/grpc_access_log_impl.h | 65 +++++-------------- 2 files changed, 18 insertions(+), 51 deletions(-) diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD index 25c7a9c3aed13..f4e9c1ffed246 100644 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD +++ b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD @@ -1,9 +1,9 @@ -# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. - load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + api_proto_package( deps = [ "//envoy/config/accesslog/v3:pkg", diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 347775354098d..b13c3ccf534c3 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -77,62 +77,17 @@ class GrpcCriticalAccessLogClient { Grpc::BufferedAsyncClient& client, std::chrono::milliseconds message_ack_timeout) : dispatcher_(dispatcher), message_ack_timeout_(message_ack_timeout), stats_(stats), - client_(client) {} + client_(client), timer_(dispatcher_.createTimer([this] { callback(); })) {} - ~InflightMessageTtlManager() { - if (timer_ != nullptr) { - timer_->disableTimer(); - } - } + ~InflightMessageTtlManager() { timer_->disableTimer(); } void setDeadline(absl::flat_hash_set&& ids) { auto expires_at = dispatcher_.timeSource().monotonicTime() + message_ack_timeout_; deadline_.emplace(expires_at, std::move(ids)); - - if (timer_ == nullptr) { - timer_ = dispatcher_.createTimer([this] { - callback(); - restartTimer(); - }); - - // Since this if section is never called except at the first startup, - // it is sufficient to set message_ack_timeout. - setTimer(message_ack_timeout_); - } + timer_->enableTimer(message_ack_timeout_); } private: - void restartTimer() { - if (timer_ != nullptr) { - return; - } - - if (deadline_.empty()) { - timer_ = nullptr; - return; - } - - // When restarting the timer, set the earliest time point among the currently remaining - // messages. This will allow you to enforce an accurate timeout. - const auto earlier_timepoint = deadline_.rbegin()->first; - auto timer_duration = std::chrono::duration_cast( - earlier_timepoint - dispatcher_.timeSource().monotonicTime()); - - // A corner case may occur where the timer is not subject to a timeout - // at the stage where it fires, but becomes subject to a timeout at the - // stage where the timer glue start takes place. This if statement is - // a mechanism to prevent this from happening. - setTimer(timer_duration.count() > 0 ? timer_duration : message_ack_timeout_); - } - - void setTimer(std::chrono::milliseconds duration) { - timer_ = dispatcher_.createTimer([this] { - callback(); - restartTimer(); - }); - timer_->enableTimer(duration); - } - void callback() { const auto now = dispatcher_.timeSource().monotonicTime(); std::vector expired_timepoints; @@ -153,6 +108,16 @@ class GrpcCriticalAccessLogClient { deadline_.erase(timepoint); } + std::chrono::milliseconds timer_duration; + + if (!deadline_.empty()) { + // When restarting the timer, set the earliest time point among the currently remaining + // messages. This will allow you to enforce an accurate timeout. + const auto earliest_timepoint = deadline_.rbegin()->first; + timer_duration = std::chrono::duration_cast( + earliest_timepoint - dispatcher_.timeSource().monotonicTime()); + } + // Restore pending messages to buffer due to timeout. for (auto&& id : expired_message_ids) { const auto& message_buffer = client_.messageBuffer(); @@ -168,7 +133,9 @@ class GrpcCriticalAccessLogClient { } } - timer_ = nullptr; + if (!deadline_.empty()) { + timer_->enableTimer(timer_duration); + } } Event::Dispatcher& dispatcher_; From b7c409eb2bfbaeb7f7d65d703cb33c837883359e Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 15 Sep 2021 15:56:26 +0000 Subject: [PATCH 34/39] fix Signed-off-by: Shikugawa --- .../common/grpc_access_logger_test.cc | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/test/extensions/access_loggers/common/grpc_access_logger_test.cc b/test/extensions/access_loggers/common/grpc_access_logger_test.cc index 36781ff451db9..00b76b2adaf88 100644 --- a/test/extensions/access_loggers/common/grpc_access_logger_test.cc +++ b/test/extensions/access_loggers/common/grpc_access_logger_test.cc @@ -49,12 +49,11 @@ class MockGrpcAccessLoggerImpl ProtobufWkt::Struct> { public: MockGrpcAccessLoggerImpl(const Grpc::RawAsyncClientSharedPtr& client, - std::chrono::milliseconds buffer_flush_interval_msec, - uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, - Stats::Scope& scope, std::string access_log_prefix, + uint64_t max_buffer_size_bytes, Stats::Scope& scope, + std::string access_log_prefix, const Protobuf::MethodDescriptor& service_method) - : GrpcAccessLogger(std::move(client), buffer_flush_interval_msec, max_buffer_size_bytes, - dispatcher, scope, access_log_prefix, service_method) {} + : GrpcAccessLogger(std::move(client), max_buffer_size_bytes, scope, access_log_prefix, + service_method) {} int numInits() const { return num_inits_; } @@ -115,8 +114,9 @@ class GrpcAccessLogTest : public testing::Test { timer_ = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*timer_, enableTimer(buffer_flush_interval_msec, _)); logger_ = std::make_unique( - Grpc::RawAsyncClientPtr{async_client_}, buffer_flush_interval_msec, buffer_size_bytes, - dispatcher_, stats_store_, "mock_access_log_prefix.", mockMethodDescriptor()); + Grpc::RawAsyncClientPtr{async_client_}, buffer_size_bytes, stats_store_, + "mock_access_log_prefix.", mockMethodDescriptor()); + logger_->startIntervalFlushTimer(dispatcher_, buffer_flush_interval_msec); } void expectStreamStart(MockAccessLogStream& stream, AccessLogCallbacks** callbacks_to_set) { @@ -322,9 +322,11 @@ class MockGrpcAccessLoggerCache const Grpc::RawAsyncClientSharedPtr& client, std::chrono::milliseconds buffer_flush_interval_msec, uint64_t max_buffer_size_bytes, Event::Dispatcher& dispatcher, Stats::Scope& scope) override { - return std::make_shared( - std::move(client), buffer_flush_interval_msec, max_buffer_size_bytes, dispatcher, scope, - "mock_access_log_prefix.", mockMethodDescriptor()); + auto logger = std::make_shared( + std::move(client), max_buffer_size_bytes, scope, "mock_access_log_prefix.", + mockMethodDescriptor()); + logger->startIntervalFlushTimer(dispatcher, buffer_flush_interval_msec); + return logger; } }; From f6b94510e0cbf4740f0db4a5a6bc33e487c26c29 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Thu, 16 Sep 2021 02:54:41 +0000 Subject: [PATCH 35/39] fix Signed-off-by: Shikugawa --- .../access_loggers/grpc/grpc_access_log_impl.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index b13c3ccf534c3..2a6b4f72474d5 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -108,14 +108,16 @@ class GrpcCriticalAccessLogClient { deadline_.erase(timepoint); } - std::chrono::milliseconds timer_duration; + std::chrono::milliseconds next_timer_duration; + bool schedule_next_timer = false; if (!deadline_.empty()) { // When restarting the timer, set the earliest time point among the currently remaining // messages. This will allow you to enforce an accurate timeout. const auto earliest_timepoint = deadline_.rbegin()->first; - timer_duration = std::chrono::duration_cast( + next_timer_duration = std::chrono::duration_cast( earliest_timepoint - dispatcher_.timeSource().monotonicTime()); + schedule_next_timer = true; } // Restore pending messages to buffer due to timeout. @@ -133,8 +135,8 @@ class GrpcCriticalAccessLogClient { } } - if (!deadline_.empty()) { - timer_->enableTimer(timer_duration); + if (schedule_next_timer) { + timer_->enableTimer(next_timer_duration); } } From 6f673a17defec68d78881803f712b3f836650465 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Thu, 16 Sep 2021 05:32:06 +0000 Subject: [PATCH 36/39] fix Signed-off-by: Shikugawa --- .../access_loggers/grpc/grpc_access_log_impl.h | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 2a6b4f72474d5..b537fc5b87db9 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -108,18 +108,6 @@ class GrpcCriticalAccessLogClient { deadline_.erase(timepoint); } - std::chrono::milliseconds next_timer_duration; - bool schedule_next_timer = false; - - if (!deadline_.empty()) { - // When restarting the timer, set the earliest time point among the currently remaining - // messages. This will allow you to enforce an accurate timeout. - const auto earliest_timepoint = deadline_.rbegin()->first; - next_timer_duration = std::chrono::duration_cast( - earliest_timepoint - dispatcher_.timeSource().monotonicTime()); - schedule_next_timer = true; - } - // Restore pending messages to buffer due to timeout. for (auto&& id : expired_message_ids) { const auto& message_buffer = client_.messageBuffer(); @@ -135,8 +123,10 @@ class GrpcCriticalAccessLogClient { } } - if (schedule_next_timer) { - timer_->enableTimer(next_timer_duration); + if (!deadline_.empty()) { + const auto earliest_timepoint = deadline_.rbegin()->first; + timer_->enableTimer( + std::chrono::duration_cast(earliest_timepoint - now)); } } From 14e8ad749f273fafe638fa359c4ef19bac8c763d Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Wed, 22 Sep 2021 15:18:30 +0000 Subject: [PATCH 37/39] fix Signed-off-by: Shikugawa --- .../grpc/grpc_access_log_impl.h | 52 ++++++++----------- .../grpc/grpc_access_log_impl_test.cc | 1 + 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index b537fc5b87db9..0dc8a68c4a4cd 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -82,46 +82,40 @@ class GrpcCriticalAccessLogClient { ~InflightMessageTtlManager() { timer_->disableTimer(); } void setDeadline(absl::flat_hash_set&& ids) { - auto expires_at = dispatcher_.timeSource().monotonicTime() + message_ack_timeout_; + const auto expires_at = dispatcher_.timeSource().monotonicTime() + message_ack_timeout_; deadline_.emplace(expires_at, std::move(ids)); - timer_->enableTimer(message_ack_timeout_); + + if (!timer_->enabled()) { + timer_->enableTimer(message_ack_timeout_); + } } private: void callback() { const auto now = dispatcher_.timeSource().monotonicTime(); - std::vector expired_timepoints; - absl::flat_hash_set expired_message_ids; - // Extract timeout message ids. - auto it = deadline_.lower_bound(now); - while (it != deadline_.end()) { + auto begin_it = deadline_.lower_bound(now); + const auto& message_buffer = client_.messageBuffer(); + + for (auto it = begin_it; it != deadline_.end(); ++it) { for (auto&& id : it->second) { - expired_message_ids.emplace(id); + const auto& message_it = message_buffer.find(id); + + if (message_it == message_buffer.end()) { + continue; + } + + // If the retrieved message is a PendingFlush, it means that the message + // has timed out. A timeout is treated as an error, and the callback will + // re-buffer the message. + if (message_it->second.first == Grpc::BufferState::PendingFlush) { + client_.onError(id); + stats_.critical_logs_message_timeout_.inc(); + } } - expired_timepoints.push_back(it->first); - ++it; - } - - // Clear buffered message ids on the set of waiting timeout. - for (auto&& timepoint : expired_timepoints) { - deadline_.erase(timepoint); } - // Restore pending messages to buffer due to timeout. - for (auto&& id : expired_message_ids) { - const auto& message_buffer = client_.messageBuffer(); - - if (message_buffer.find(id) == message_buffer.end()) { - continue; - } - - auto& message = message_buffer.at(id); - if (message.first == Grpc::BufferState::PendingFlush) { - client_.onError(id); - stats_.critical_logs_message_timeout_.inc(); - } - } + deadline_.erase(begin_it, deadline_.end()); if (!deadline_.empty()) { const auto earliest_timepoint = deadline_.rbegin()->first; diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index d3a06e0ccdebe..7adf575d9ca72 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -148,6 +148,7 @@ class CriticalGrpcAccessLoggerImplTest : public GrpcAccessLoggerImplTest { public: CriticalGrpcAccessLoggerImplTest() { mock_buffer_timer_ = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*mock_buffer_timer_, enabled()); EXPECT_CALL(*mock_buffer_timer_, enableTimer(_, _)); EXPECT_CALL(*mock_buffer_timer_, disableTimer()); } From 65e2d41fea9b2095a1746ab31c82b065d1769e1f Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Fri, 12 Nov 2021 17:56:13 +0900 Subject: [PATCH 38/39] fix Signed-off-by: Shikugawa --- docs/root/version_history/current.rst | 29 --------------------------- test/test_common/utility.cc | 20 ++++-------------- 2 files changed, 4 insertions(+), 45 deletions(-) diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 2da9ef77fc0ac..bc38607690c46 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -50,35 +50,6 @@ Removed Config or Runtime New Features ------------ * access log: added :ref:`critical log message ` to AccessLogService to guarantee log arrival. -* access_log: added :ref:`METADATA` token to handle all types of metadata (DYNAMIC, CLUSTER, ROUTE). -* bootstrap: added :ref:`inline_headers ` in the bootstrap to make custom inline headers bootstrap configurable. -* contrib: added new :ref:`contrib images ` which contain contrib extensions. -* dns: added :ref:`V4_PREFERRED ` option to return V6 addresses only if V4 addresses are not available. -* grpc reverse bridge: added a new :ref:`option ` to support streaming response bodies when withholding gRPC frames from the upstream. -* http: added cluster_header in :ref:`weighted_clusters ` to allow routing to the weighted cluster specified in the request_header. -* http: added :ref:`alternate_protocols_cache_options ` for enabling HTTP/3 connections to servers which advertise HTTP/3 support via `HTTP Alternative Services `_. -* http: added :ref:`string_match ` in the header matcher. -* http: added :ref:`x-envoy-upstream-stream-duration-ms ` that allows configuring the max stream duration via a request header. -* http: added support for :ref:`max_requests_per_connection ` for both upstream and downstream connections. -* http: sanitizing the referer header as documented :ref:`here `. This feature can be temporarily turned off by setting runtime guard ``envoy.reloadable_features.sanitize_http_header_referer`` to false. -* http: validating outgoing HTTP/2 CONNECT requests to ensure that if ``:path`` is set that ``:protocol`` is present. This behavior can be temporarily turned off by setting runtime guard ``envoy.reloadable_features.validate_connect`` to false. -* jwt_authn: added support for :ref:`Jwt Cache ` and its size can be specified by :ref:`jwt_cache_size `. -* jwt_authn: added support for extracting JWTs from request cookies using :ref:`from_cookies `. -* jwt_authn: added support for setting the extracted headers from a successfully verified JWT using :ref:`header_in_metadata ` to dynamic metadata. -* listener: new listener metric ``downstream_cx_transport_socket_connect_timeout`` to track transport socket timeouts. -* lua: added ``header:getAtIndex()`` and ``header:getNumValues()`` methods to :ref:`header object ` for retrieving the value of a header at certain index and get the total number of values for a given header. -* matcher: added :ref:`invert ` for inverting the match result in the metadata matcher. -* overload: add a new overload action that resets streams using a lot of memory. To enable the tracking of allocated bytes in buffers that a stream is using we need to configure the minimum threshold for tracking via:ref:`buffer_factory_config `. We have an overload action ``Envoy::Server::OverloadActionNameValues::ResetStreams`` that takes advantage of the tracking to reset the most expensive stream first. -* rbac: added :ref:`destination_port_range ` for matching range of destination ports. -* route config: added :ref:`dynamic_metadata ` for routing based on dynamic metadata. -* router: added retry options predicate extensions configured via - :ref:` `. These - extensions allow modification of requests between retries at the router level. There are not - currently any built-in extensions that implement this extension point. -* router: added :ref:`per_try_idle_timeout ` timeout configuration. -* router: added an optional :ref:`override_auto_sni_header ` to support setting SNI value from an arbitrary header other than host/authority. -* sxg_filter: added filter to transform response to SXG package to :ref:`contrib images `. This can be enabled by setting :ref:`SXG ` configuration. -* thrift_proxy: added support for :ref:`mirroring requests `. * access log: added :ref:`grpc_stream_retry_policy ` to the gRPC logger to reconnect when a connection fails to be established. * api: added support for *xds.type.v3.TypedStruct* in addition to the now-deprecated *udpa.type.v1.TypedStruct* proto message, which is a wrapper proto used to encode typed JSON data in a *google.protobuf.Any* field. * bootstrap: added :ref:`typed_dns_resolver_config ` in the bootstrap to support DNS resolver as an extension. diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 90b74ee5396d4..865d84f3aa05c 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -178,11 +178,8 @@ AssertionResult TestUtility::waitForCounterEq(Stats::Store& store, const std::st std::chrono::milliseconds timeout, Event::Dispatcher* dispatcher) { Event::TestTimeSystem::RealTimeBound bound(timeout); - while (true) { + while (findCounter(store, name) == nullptr || findCounter(store, name)->value() != value) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); - if (findCounter(store, name) != nullptr && findCounter(store, name)->value() == value) { - break; - } if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { std::string current_value; if (findCounter(store, name)) { @@ -204,11 +201,8 @@ AssertionResult TestUtility::waitForCounterGe(Stats::Store& store, const std::st uint64_t value, Event::TestTimeSystem& time_system, std::chrono::milliseconds timeout) { Event::TestTimeSystem::RealTimeBound bound(timeout); - while (true) { + while (findCounter(store, name) == nullptr || findCounter(store, name)->value() < value) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); - if (findCounter(store, name) != nullptr && findCounter(store, name)->value() >= value) { - break; - } if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { return AssertionFailure() << fmt::format("timed out waiting for {} to be {}", name, value); } @@ -220,11 +214,8 @@ AssertionResult TestUtility::waitForGaugeGe(Stats::Store& store, const std::stri uint64_t value, Event::TestTimeSystem& time_system, std::chrono::milliseconds timeout) { Event::TestTimeSystem::RealTimeBound bound(timeout); - while (true) { + while (findGauge(store, name) == nullptr || findGauge(store, name)->value() < value) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); - if (findGauge(store, name) != nullptr && findGauge(store, name)->value() >= value) { - break; - } if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { return AssertionFailure() << fmt::format("timed out waiting for {} to be {}", name, value); } @@ -236,11 +227,8 @@ AssertionResult TestUtility::waitForGaugeEq(Stats::Store& store, const std::stri uint64_t value, Event::TestTimeSystem& time_system, std::chrono::milliseconds timeout) { Event::TestTimeSystem::RealTimeBound bound(timeout); - while (true) { + while (findGauge(store, name) == nullptr || findGauge(store, name)->value() != value) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); - if (findGauge(store, name) != nullptr && findGauge(store, name)->value() == value) { - break; - } if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { std::string current_value; if (findGauge(store, name)) { From 3feffd84dd02fda847ef2a76e02d626fae892179 Mon Sep 17 00:00:00 2001 From: Shikugawa Date: Fri, 12 Nov 2021 17:58:23 +0900 Subject: [PATCH 39/39] fix Signed-off-by: Shikugawa --- .../extensions/access_loggers/grpc/v3/BUILD | 13 -- .../access_loggers/grpc/v3/als.proto | 106 ------------- .../envoy/service/accesslog/v3/als.proto | 145 ------------------ 3 files changed, 264 deletions(-) delete mode 100644 generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD delete mode 100644 generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto delete mode 100644 generated_api_shadow/envoy/service/accesslog/v3/als.proto diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD deleted file mode 100644 index f4e9c1ffed246..0000000000000 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") - -licenses(["notice"]) # Apache 2 - -# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. - -api_proto_package( - deps = [ - "//envoy/config/accesslog/v3:pkg", - "//envoy/config/core/v3:pkg", - "@com_github_cncf_udpa//udpa/annotations:pkg", - ], -) diff --git a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto b/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto deleted file mode 100644 index 29904142368e1..0000000000000 --- a/generated_api_shadow/envoy/extensions/access_loggers/grpc/v3/als.proto +++ /dev/null @@ -1,106 +0,0 @@ -syntax = "proto3"; - -package envoy.extensions.access_loggers.grpc.v3; - -import "envoy/config/accesslog/v3/accesslog.proto"; -import "envoy/config/core/v3/config_source.proto"; -import "envoy/config/core/v3/grpc_service.proto"; - -import "google/protobuf/duration.proto"; -import "google/protobuf/wrappers.proto"; - -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; - -option java_package = "io.envoyproxy.envoy.extensions.access_loggers.grpc.v3"; -option java_outer_classname = "AlsProto"; -option java_multiple_files = true; -option (udpa.annotations.file_status).package_version_status = ACTIVE; - -// [#protodoc-title: gRPC Access Log Service (ALS)] - -// Configuration for the built-in *envoy.access_loggers.http_grpc* -// :ref:`AccessLog `. This configuration will -// populate :ref:`StreamAccessLogsMessage.http_logs -// `. -// [#extension: envoy.access_loggers.http_grpc] -message HttpGrpcAccessLogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.accesslog.v2.HttpGrpcAccessLogConfig"; - - CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message = {required: true}]; - - // Additional request headers to log in :ref:`HTTPRequestProperties.request_headers - // `. - repeated string additional_request_headers_to_log = 2; - - // Additional response headers to log in :ref:`HTTPResponseProperties.response_headers - // `. - repeated string additional_response_headers_to_log = 3; - - // Additional response trailers to log in :ref:`HTTPResponseProperties.response_trailers - // `. - repeated string additional_response_trailers_to_log = 4; -} - -// Configuration for the built-in *envoy.access_loggers.tcp_grpc* type. This configuration will -// populate *StreamAccessLogsMessage.tcp_logs*. -// [#extension: envoy.access_loggers.tcp_grpc] -message TcpGrpcAccessLogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.accesslog.v2.TcpGrpcAccessLogConfig"; - - CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message = {required: true}]; -} - -// Common configuration for gRPC access logs. -// [#next-free-field: 10] -message CommonGrpcAccessLogConfig { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.accesslog.v2.CommonGrpcAccessLogConfig"; - - // The friendly name of the access log to be returned in :ref:`StreamAccessLogsMessage.Identifier - // `. This allows the - // access log server to differentiate between different access logs coming from the same Envoy. - string log_name = 1 [(validate.rules).string = {min_len: 1}]; - - // The gRPC service for the access log service. - config.core.v3.GrpcService grpc_service = 2 [(validate.rules).message = {required: true}]; - - // API version for access logs service transport protocol. This describes the access logs service - // gRPC endpoint and version of messages used on the wire. - config.core.v3.ApiVersion transport_api_version = 6 - [(validate.rules).enum = {defined_only: true}]; - - // Interval for flushing access logs to the gRPC stream. Logger will flush requests every time - // this interval is elapsed, or when batch size limit is hit, whichever comes first. Defaults to - // 1 second. - google.protobuf.Duration buffer_flush_interval = 3 [(validate.rules).duration = {gt {}}]; - - // Soft size limit in bytes for access log entries buffer. Logger will buffer requests until - // this limit it hit, or every time flush interval is elapsed, whichever comes first. Setting it - // to zero effectively disables the batching. Defaults to 16384. - google.protobuf.UInt32Value buffer_size_bytes = 4; - - // Additional filter state objects to log in :ref:`filter_state_objects - // `. - // Logger will call `FilterState::Object::serializeAsProto` to serialize the filter state object. - repeated string filter_state_objects_to_log = 5; - - // Define the log condition for critical access logs. - // Logs that match the filter are not forwarded to the normal endpoint, - // but are sent to a special endpoint. - config.accesslog.v3.AccessLogFilter critical_buffer_log_filter = 7; - - // The time to wait for an ACK message. If no ACK message is returned after this time, - // the message is considered undeliverable and the failed transmission is buffered again. - // The re-buffered message will be sent again at the next time a log matching *critical_buffer_log_filter* is queued. - google.protobuf.Duration message_ack_timeout = 8 - [(validate.rules).duration = {gte {nanos: 1000000}}]; - - // Size limit (in bytes) of the buffer used to store messages during processing in a - // critical logger. A critical logger buffers messages until it receives an ACK from upstream. - // The default is 16384. - google.protobuf.UInt32Value max_pending_buffer_size_bytes = 9; -} diff --git a/generated_api_shadow/envoy/service/accesslog/v3/als.proto b/generated_api_shadow/envoy/service/accesslog/v3/als.proto deleted file mode 100644 index 4f594130c2d3b..0000000000000 --- a/generated_api_shadow/envoy/service/accesslog/v3/als.proto +++ /dev/null @@ -1,145 +0,0 @@ -syntax = "proto3"; - -package envoy.service.accesslog.v3; - -import "envoy/config/core/v3/base.proto"; -import "envoy/data/accesslog/v3/accesslog.proto"; - -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; - -option java_package = "io.envoyproxy.envoy.service.accesslog.v3"; -option java_outer_classname = "AlsProto"; -option java_multiple_files = true; -option java_generic_services = true; -option (udpa.annotations.file_status).package_version_status = ACTIVE; - -// [#protodoc-title: gRPC Access Log Service (ALS)] - -// Service for streaming access logs from Envoy to an access log server. -service AccessLogService { - // Envoy will connect and send StreamAccessLogsMessage messages forever. It does not expect any - // response to be sent as nothing would be done in the case of failure. The server should - // disconnect if it expects Envoy to reconnect. This API is designed for high throughput with the - // expectation that it might be lossy. - rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { - } - - // This endpoint provides acknowledgment of logs marked as requiring acknowledgment. - // The requirement for an acknowledgment can be set in - // :ref:`critical_buffer_log_filter `. - // Log messages that match this filter will be guaranteed delivery. In order to guarantee - // the arrival, this endpoint performs the following process. - // - // 1. A response message is returned for each log. The response message includes ACK/NACK status, - // and in case of NACK, the target log is not flushed but buffered by Envoy. - // 2. Timeout for response message is set and if no message is returned within a certain time, - // it will be considered as unreachable and buffered by Envoy without flushing the target log. This timeout is set by - // :ref:`message_ack_timeout `. - // - // On the ALS receiver side, ACK is expected to be returned to indicate that the log was saved properly, - // and NACK is expected to be returned when the log could not be saved due to some error. - // - // .. attention:: - // - // Buffers for guaranteed reachability can be extremely memory-intensive. Therefore, the following points - // should be considered when using this endpoint. - // - // 1. :ref:`critical_buffer_log_filter ` - // should be set strictly. A loose filter may encourage rapid buffer overwhelm and leading to OOM. - // 2. :ref:`max_pending_buffer_size_bytes ` - // should be set appropriately to prevent OOM. - // 3. Make sure that ALS receiver is implemented properly. If it is not implemented, all messages will - // be buffered, which may cause OOM soon. - rpc CriticalAccessLogs(stream CriticalAccessLogsMessage) - returns (stream CriticalAccessLogsResponse) { - } -} - -// Empty response for the StreamAccessLogs API. Will never be sent. See below. -message StreamAccessLogsResponse { - option (udpa.annotations.versioning).previous_message_type = - "envoy.service.accesslog.v2.StreamAccessLogsResponse"; -} - -// Response received to identify undelivered or delivered messages in CriticalAccessLogs. -message CriticalAccessLogsResponse { - enum Status { - // Indicates that the message has been received. - ACK = 0; - - // Indicates that the message has not been received. - NACK = 1; - } - - // This field is used to indicate the arrival status. - Status status = 1; - - // Message ID that identifies a message. - uint64 id = 2; -} - -// Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream -// access logs without ever expecting a response. -message StreamAccessLogsMessage { - option (udpa.annotations.versioning).previous_message_type = - "envoy.service.accesslog.v2.StreamAccessLogsMessage"; - - message Identifier { - option (udpa.annotations.versioning).previous_message_type = - "envoy.service.accesslog.v2.StreamAccessLogsMessage.Identifier"; - - // The node sending the access log messages over the stream. - config.core.v3.Node node = 1 [(validate.rules).message = {required: true}]; - - // The friendly name of the log configured in :ref:`CommonGrpcAccessLogConfig - // `. - string log_name = 2 [(validate.rules).string = {min_len: 1}]; - } - - // Wrapper for batches of HTTP access log entries. - message HTTPAccessLogEntries { - option (udpa.annotations.versioning).previous_message_type = - "envoy.service.accesslog.v2.StreamAccessLogsMessage.HTTPAccessLogEntries"; - - repeated data.accesslog.v3.HTTPAccessLogEntry log_entry = 1 - [(validate.rules).repeated = {min_items: 1}]; - } - - // Wrapper for batches of TCP access log entries. - message TCPAccessLogEntries { - option (udpa.annotations.versioning).previous_message_type = - "envoy.service.accesslog.v2.StreamAccessLogsMessage.TCPAccessLogEntries"; - - repeated data.accesslog.v3.TCPAccessLogEntry log_entry = 1 - [(validate.rules).repeated = {min_items: 1}]; - } - - // Identifier data that will only be sent in the first message on the stream. This is effectively - // structured metadata and is a performance optimization. - Identifier identifier = 1; - - // Batches of log entries of a single type. Generally speaking, a given stream should only - // ever include one type of log entry. - oneof log_entries { - option (validate.required) = true; - - HTTPAccessLogEntries http_logs = 2; - - TCPAccessLogEntries tcp_logs = 3; - } -} - -// Stream message for the CriticalAccessLogs API. -// Envoy opens a stream to the server and streams the access log, -// expecting a response. Each message sent is assigned an individual ID, -// and the state of the message is tracked based on the ID. -message CriticalAccessLogsMessage { - // The body of the log message sent to CriticalAccessLogs. - StreamAccessLogsMessage message = 1; - - // This is an ID to identify the message, and should be added to the Critical Endpoint - // response message to uniquely identify the message being ACK/NACKed. - uint64 id = 4; -}