Skip to content

Implementing drop_overload stats reporting in load report service API #31384

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yanavlasov merged 28 commits into from
Dec 16, 2023
Merged

Implementing drop_overload stats reporting in load report service API #31384

Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
1cf79c8
Support drop_overload API.
yanjunxiang-google Nov 10, 2023
b683739
only support the first category
yanjunxiang-google Nov 12, 2023
197c20c
moving the drop overload API to cluster
yanjunxiang-google Nov 22, 2023
89eb945
adding tests
yanjunxiang-google Nov 24, 2023
4b7e6d1
adding more tests
yanjunxiang-google Nov 28, 2023
38a553e
adding change logs
yanjunxiang-google Nov 28, 2023
f2ca31a
changing API doc
yanjunxiang-google Nov 28, 2023
25f4303
addressing comments
yanjunxiang-google Nov 29, 2023
987a17b
addressing comments
yanjunxiang-google Nov 29, 2023
9962114
adding tests
yanjunxiang-google Nov 29, 2023
a15d301
Merge branch 'main' of https://github.com/envoyproxy/envoy into drop_…
yanjunxiang-google Nov 29, 2023
151565a
addressing comments
yanjunxiang-google Nov 29, 2023
90bc89a
adding tests in router_test.cc
yanjunxiang-google Nov 30, 2023
281d232
addressing comments
yanjunxiang-google Nov 30, 2023
7934052
adding config dump for drop_overload
yanjunxiang-google Dec 1, 2023
4d53b69
fixing the CI coverage error
yanjunxiang-google Dec 1, 2023
71b556d
moving drop_overload from cluster trafficStats to loadReportStats
yanjunxiang-google Dec 1, 2023
b2c1fa9
addressing comments
yanjunxiang-google Dec 4, 2023
5df585d
Empty-Commit
yanjunxiang-google Dec 5, 2023
29a8136
moving the drop overload check into a private method
yanjunxiang-google Dec 5, 2023
c67db02
removing unncessary line change in Envoy stream logs
yanjunxiang-google Dec 5, 2023
96c7ab5
merge upstream main
yanjunxiang-google Dec 13, 2023
94d2ff1
Implementing drop_overload support for Envoy load report service API
yanjunxiang-google Dec 14, 2023
7660398
adding const
yanjunxiang-google Dec 15, 2023
ad4c840
removing the global test flag
yanjunxiang-google Dec 15, 2023
d5d6647
removing unncecessary changes
yanjunxiang-google Dec 15, 2023
5f2fc0c
fixing format issue
yanjunxiang-google Dec 15, 2023
69e7023
Empty-Commit
yanjunxiang-google Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions source/common/upstream/load_stats_reporter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,13 @@ void LoadStatsReporter::sendLoadStatsRequest() {
}
cluster_stats->set_total_dropped_requests(
cluster.info()->loadReportStats().upstream_rq_dropped_.latch());
uint64_t drop_overload_count = cluster.info()->loadReportStats().upstream_rq_drop_overload_.latch();
Comment thread
yanjunxiang-google marked this conversation as resolved.
Outdated
if (drop_overload_count > 0 ) {
Comment thread
yanjunxiang-google marked this conversation as resolved.
Outdated
auto* dropped_request = cluster_stats->add_dropped_requests();
dropped_request->set_category("drop_overload");
dropped_request->set_dropped_count(drop_overload_count);
}

const auto now = time_source_.monotonicTime().time_since_epoch();
const auto measured_interval = now - cluster_name_and_timestamp.second;
cluster_stats->mutable_load_report_interval()->MergeFrom(
Expand Down Expand Up @@ -214,6 +221,7 @@ void LoadStatsReporter::startLoadReportPeriod() {
}
}
cluster.info()->loadReportStats().upstream_rq_dropped_.latch();
cluster.info()->loadReportStats().upstream_rq_drop_overload_.latch();
};
if (message_->send_all_clusters()) {
for (const auto& p : all_clusters.active_clusters_) {
Expand Down
20 changes: 20 additions & 0 deletions test/common/upstream/load_stats_reporter_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,12 @@ class LoadStatsReporterTest : public testing::Test {
load_stats_reporter_->onReceiveMessage(std::move(response));
}

void setDropOverload(envoy::config::endpoint::v3::ClusterStats& cluster_stats, uint64_t count) {
auto* dropped_request = cluster_stats.add_dropped_requests();
dropped_request->set_category("drop_overload");
dropped_request->set_dropped_count(count);
}

Event::SimulatedTimeSystem time_system_;
NiceMock<Upstream::MockClusterManager> cm_;
Event::MockDispatcher dispatcher_;
Expand Down Expand Up @@ -132,12 +138,14 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) {
deliverLoadStatsResponse({"foo"});
// Initial stats report for foo on timer tick.
foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(5);
foo_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(7);
time_system_.setMonotonicTime(std::chrono::microseconds(4));
{
envoy::config::endpoint::v3::ClusterStats foo_cluster_stats;
foo_cluster_stats.set_cluster_name("foo");
foo_cluster_stats.set_cluster_service_name("bar");
foo_cluster_stats.set_total_dropped_requests(5);
setDropOverload(foo_cluster_stats, 7);
foo_cluster_stats.mutable_load_report_interval()->MergeFrom(
Protobuf::util::TimeUtil::MicrosecondsToDuration(1));
expectSendMessage({foo_cluster_stats});
Expand All @@ -148,13 +156,15 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) {
// Some traffic on foo/bar in between previous request and next response.
foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1);
bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1);
bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(5);

// Start reporting on bar.
time_system_.setMonotonicTime(std::chrono::microseconds(6));
deliverLoadStatsResponse({"foo", "bar"});
// Stats report foo/bar on timer tick.
foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1);
bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1);
bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(3);
time_system_.setMonotonicTime(std::chrono::microseconds(28));
{
envoy::config::endpoint::v3::ClusterStats foo_cluster_stats;
Expand All @@ -166,6 +176,7 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) {
envoy::config::endpoint::v3::ClusterStats bar_cluster_stats;
bar_cluster_stats.set_cluster_name("bar");
bar_cluster_stats.set_total_dropped_requests(1);
setDropOverload(bar_cluster_stats, 3);
bar_cluster_stats.mutable_load_report_interval()->MergeFrom(
Protobuf::util::TimeUtil::MicrosecondsToDuration(22));
expectSendMessage({bar_cluster_stats, foo_cluster_stats});
Expand All @@ -176,17 +187,20 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) {
// Some traffic on foo/bar in between previous request and next response.
foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1);
bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1);
bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(1);

// Stop reporting on foo.
deliverLoadStatsResponse({"bar"});
// Stats report for bar on timer tick.
foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(5);
bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(5);
bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(7);
time_system_.setMonotonicTime(std::chrono::microseconds(33));
{
envoy::config::endpoint::v3::ClusterStats bar_cluster_stats;
bar_cluster_stats.set_cluster_name("bar");
bar_cluster_stats.set_total_dropped_requests(6);
setDropOverload(bar_cluster_stats, 8);
bar_cluster_stats.mutable_load_report_interval()->MergeFrom(
Protobuf::util::TimeUtil::MicrosecondsToDuration(5));
expectSendMessage({bar_cluster_stats});
Expand All @@ -196,25 +210,31 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) {

// Some traffic on foo/bar in between previous request and next response.
foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1);
foo_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(8);
bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1);
bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(3);

// Start tracking foo again, we should forget earlier history for foo.
time_system_.setMonotonicTime(std::chrono::microseconds(43));
deliverLoadStatsResponse({"foo", "bar"});
// Stats report foo/bar on timer tick.
foo_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1);
foo_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(9);
bar_cluster.info_->load_report_stats_.upstream_rq_dropped_.add(1);
bar_cluster.info_->load_report_stats_.upstream_rq_drop_overload_.add(4);
time_system_.setMonotonicTime(std::chrono::microseconds(47));
{
envoy::config::endpoint::v3::ClusterStats foo_cluster_stats;
foo_cluster_stats.set_cluster_name("foo");
foo_cluster_stats.set_cluster_service_name("bar");
foo_cluster_stats.set_total_dropped_requests(1);
setDropOverload(foo_cluster_stats, 9);
foo_cluster_stats.mutable_load_report_interval()->MergeFrom(
Protobuf::util::TimeUtil::MicrosecondsToDuration(4));
envoy::config::endpoint::v3::ClusterStats bar_cluster_stats;
bar_cluster_stats.set_cluster_name("bar");
bar_cluster_stats.set_total_dropped_requests(2);
setDropOverload(bar_cluster_stats, 7);
bar_cluster_stats.mutable_load_report_interval()->MergeFrom(
Protobuf::util::TimeUtil::MicrosecondsToDuration(14));
expectSendMessage({bar_cluster_stats, foo_cluster_stats});
Expand Down
83 changes: 62 additions & 21 deletions test/integration/load_stats_integration_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,17 @@ class LoadStatsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest,
const LocalityAssignment& p1_dragon_upstreams) {
uint32_t num_endpoints = 0;
envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment;

if (drop_overload_test_) {
// Config drop_overload to drop 100% requests.
auto* policy = cluster_load_assignment.mutable_policy();
auto* drop_overload = policy->add_drop_overloads();
drop_overload->set_category("drop_overload");
auto* drop_percentage = drop_overload->mutable_drop_percentage();
drop_percentage->set_numerator(100);
drop_percentage->set_denominator(envoy::type::v3::FractionalPercent::HUNDRED);
}
Comment thread
yanjunxiang-google marked this conversation as resolved.
Outdated

// EDS service_name is set in cluster_0
cluster_load_assignment.set_cluster_name("service_name_0");

Expand Down Expand Up @@ -191,6 +202,20 @@ class LoadStatsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest,

cluster_stats->set_total_dropped_requests(cluster_stats->total_dropped_requests() +
local_cluster_stats.total_dropped_requests());
if (local_cluster_stats.dropped_requests().size() > 0) {
uint64_t local_drop_count = local_cluster_stats.dropped_requests(0).dropped_count();
Comment thread
yanjunxiang-google marked this conversation as resolved.
Outdated
if (local_drop_count > 0 ) {
envoy::config::endpoint::v3::ClusterStats::DroppedRequests* drop_request;
if (cluster_stats->dropped_requests().size() > 0) {
drop_request = cluster_stats->mutable_dropped_requests(0);
drop_request->set_dropped_count(drop_request->dropped_count() + local_drop_count);
} else {
drop_request = cluster_stats->add_dropped_requests();
drop_request->set_dropped_count(local_drop_count);
}
drop_request->set_category("drop_overload");
}
}

for (int i = 0; i < local_cluster_stats.upstream_locality_stats_size(); ++i) {
const auto& local_upstream_locality_stats = local_cluster_stats.upstream_locality_stats(i);
Expand Down Expand Up @@ -253,6 +278,11 @@ class LoadStatsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest,
cluster_stats->set_cluster_service_name("service_name_0");
if (dropped > 0) {
cluster_stats->set_total_dropped_requests(dropped);
if (drop_overload_test_) {
auto* drop_request = cluster_stats->add_dropped_requests();
drop_request->set_category("drop_overload");
drop_request->set_dropped_count(dropped);
}
}
std::copy(
expected_locality_stats.begin(), expected_locality_stats.end(),
Expand Down Expand Up @@ -368,6 +398,31 @@ class LoadStatsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest,
cleanupUpstreamAndDownstream();
}

void dropLoadTest() {
initialize();

waitForLoadStatsStream();
ASSERT_TRUE(waitForLoadStatsRequest({}));
loadstats_stream_->startGrpcStream();

updateClusterLoadAssignment({{0}}, {}, {}, {});
requestLoadStatsResponse({"cluster_0"});
initiateClientConnection();
ASSERT_TRUE(response_->waitForEndStream());
ASSERT_TRUE(response_->complete());
EXPECT_EQ("503", response_->headers().getStatusValue());
cleanupUpstreamAndDownstream();

ASSERT_TRUE(waitForLoadStatsRequest({}, 1));

EXPECT_EQ(1, test_server_->counter("load_reporter.requests")->value());
EXPECT_LE(2, test_server_->counter("load_reporter.responses")->value());
EXPECT_EQ(0, test_server_->counter("load_reporter.errors")->value());

cleanupLoadStatsConnection();
}


static constexpr uint32_t upstream_endpoints_ = 5;

IntegrationStreamDecoderPtr response_;
Expand All @@ -379,6 +434,7 @@ class LoadStatsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest,
uint32_t load_requests_{};
EdsHelper eds_helper_;
bool locality_weighted_lb_{};
bool drop_overload_test_{false};
Comment thread
yanjunxiang-google marked this conversation as resolved.
Outdated

const uint64_t request_size_ = 1024;
const uint64_t response_size_ = 512;
Expand Down Expand Up @@ -622,28 +678,13 @@ TEST_P(LoadStatsIntegrationTest, Dropped) {
auto* thresholds = cluster_0->mutable_circuit_breakers()->add_thresholds();
thresholds->mutable_max_pending_requests()->set_value(0);
});
initialize();

waitForLoadStatsStream();
ASSERT_TRUE(waitForLoadStatsRequest({}));
loadstats_stream_->startGrpcStream();

updateClusterLoadAssignment({{0}}, {}, {}, {});
requestLoadStatsResponse({"cluster_0"});
// This should count as dropped, since we trigger circuit breaking.
initiateClientConnection();
ASSERT_TRUE(response_->waitForEndStream());
ASSERT_TRUE(response_->complete());
EXPECT_EQ("503", response_->headers().getStatusValue());
cleanupUpstreamAndDownstream();

ASSERT_TRUE(waitForLoadStatsRequest({}, 1));

EXPECT_EQ(1, test_server_->counter("load_reporter.requests")->value());
EXPECT_LE(2, test_server_->counter("load_reporter.responses")->value());
EXPECT_EQ(0, test_server_->counter("load_reporter.errors")->value());
dropLoadTest();
}

cleanupLoadStatsConnection();
// Validate the load reports for dropped requests due to drop_overload make sense.
TEST_P(LoadStatsIntegrationTest, DropOverloadDropped) {
drop_overload_test_ = true;
dropLoadTest();
}

} // namespace
Expand Down