From 01d20022f27cb572090c639a3c9afc567dcc127a Mon Sep 17 00:00:00 2001 From: Karl Sassenberg <87377621+karlsassenberg@users.noreply.github.com> Date: Mon, 27 Mar 2023 10:58:13 +0100 Subject: [PATCH] Create dedicated cache and upstream metrics reports (#189) * Create dedicated cache and upstream metrics reports Split Nginx Plus MetricsReport into three groups of reports of the following types: CACHE_ZONE, UPSTREAMS and SYSTEM. A CACHE_ZONE report is created for each distinct value of the dimension "cache_zone". An UPSTREAMS report is created for each distinct value of the dimension "upstream". A SYSTEM report contains all other metrics. --- docs/proto/proto.md | 2 + sdk/client/metric_reporter.go | 2 +- sdk/proto/metrics.pb.go | 76 +++--- sdk/proto/metrics.proto | 4 + src/core/config/defaults.go | 2 +- src/core/metrics/aggregate.go | 10 +- src/core/metrics/aggregate_test.go | 8 +- src/core/metrics/collectors/container.go | 9 +- src/core/metrics/collectors/container_test.go | 8 +- src/core/metrics/collectors/nginx.go | 7 +- src/core/metrics/collectors/nginx_test.go | 2 +- .../metrics/collectors/source_mocks_test.go | 5 +- src/core/metrics/collectors/system.go | 9 +- src/core/metrics/collectors/system_test.go | 8 +- src/core/metrics/metrics_util.go | 51 +++- src/core/metrics/sources/common.go | 4 +- src/core/metrics/sources/common_test.go | 7 +- src/core/metrics/sources/container_cpu.go | 4 +- .../metrics/sources/container_cpu_test.go | 9 +- src/core/metrics/sources/container_mem.go | 4 +- .../metrics/sources/container_mem_test.go | 9 +- src/core/metrics/sources/cpu.go | 4 +- src/core/metrics/sources/cpu_test.go | 14 +- src/core/metrics/sources/disk.go | 4 +- src/core/metrics/sources/disk_io.go | 4 +- src/core/metrics/sources/disk_io_test.go | 6 +- src/core/metrics/sources/disk_test.go | 6 +- src/core/metrics/sources/load.go | 4 +- src/core/metrics/sources/load_test.go | 6 +- src/core/metrics/sources/mem.go | 4 +- src/core/metrics/sources/mem_test.go | 6 +- src/core/metrics/sources/net_io.go | 6 +- src/core/metrics/sources/net_io_test.go | 6 +- src/core/metrics/sources/nginx_access_log.go | 12 +- .../metrics/sources/nginx_access_log_test.go | 6 +- src/core/metrics/sources/nginx_error_log.go | 14 +- .../metrics/sources/nginx_error_log_test.go | 6 +- src/core/metrics/sources/nginx_oss.go | 4 +- src/core/metrics/sources/nginx_oss_test.go | 10 +- src/core/metrics/sources/nginx_plus.go | 87 +++---- src/core/metrics/sources/nginx_plus_test.go | 228 +++++++++--------- src/core/metrics/sources/nginx_process.go | 4 +- .../metrics/sources/nginx_process_test.go | 12 +- src/core/metrics/sources/nginx_static.go | 3 +- src/core/metrics/sources/nginx_static_test.go | 8 +- src/core/metrics/sources/nginx_worker.go | 4 +- src/core/metrics/sources/nginx_worker_test.go | 6 +- src/core/metrics/sources/swap.go | 4 +- src/core/metrics/sources/swap_test.go | 9 +- .../prometheus-metrics/prometheus_exporter.go | 23 +- .../prometheus_exporter_test.go | 13 +- src/plugins/agent_api.go | 5 +- src/plugins/agent_api_test.go | 8 +- src/plugins/metrics.go | 24 +- src/plugins/metrics_sender.go | 4 +- src/plugins/metrics_throlling.go | 82 ++++--- src/plugins/metrics_throlling_test.go | 40 ++- test/component/agent_api_test.go | 68 ++++-- test/integration/api/api_test.go | 1 - .../agent/sdk/v2/client/metric_reporter.go | 2 +- .../nginx/agent/sdk/v2/proto/metrics.pb.go | 76 +++--- .../nginx/agent/sdk/v2/proto/metrics.proto | 4 + .../agent/sdk/v2/client/metric_reporter.go | 2 +- .../nginx/agent/sdk/v2/proto/metrics.pb.go | 76 +++--- .../nginx/agent/sdk/v2/proto/metrics.proto | 4 + .../agent/v2/src/core/config/defaults.go | 2 +- .../agent/v2/src/core/metrics/aggregate.go | 10 +- .../src/core/metrics/collectors/container.go | 9 +- .../v2/src/core/metrics/collectors/nginx.go | 7 +- .../v2/src/core/metrics/collectors/system.go | 9 +- .../agent/v2/src/core/metrics/metrics_util.go | 51 +++- .../v2/src/core/metrics/sources/common.go | 4 +- .../src/core/metrics/sources/container_cpu.go | 4 +- .../src/core/metrics/sources/container_mem.go | 4 +- .../agent/v2/src/core/metrics/sources/cpu.go | 4 +- .../agent/v2/src/core/metrics/sources/disk.go | 4 +- .../v2/src/core/metrics/sources/disk_io.go | 4 +- .../agent/v2/src/core/metrics/sources/load.go | 4 +- .../agent/v2/src/core/metrics/sources/mem.go | 4 +- .../v2/src/core/metrics/sources/net_io.go | 6 +- .../core/metrics/sources/nginx_access_log.go | 12 +- .../core/metrics/sources/nginx_error_log.go | 14 +- .../v2/src/core/metrics/sources/nginx_oss.go | 4 +- .../v2/src/core/metrics/sources/nginx_plus.go | 87 +++---- .../src/core/metrics/sources/nginx_process.go | 4 +- .../src/core/metrics/sources/nginx_static.go | 3 +- .../src/core/metrics/sources/nginx_worker.go | 4 +- .../agent/v2/src/core/metrics/sources/swap.go | 4 +- .../prometheus-metrics/prometheus_exporter.go | 23 +- .../nginx/agent/v2/src/plugins/agent_api.go | 5 +- .../nginx/agent/v2/src/plugins/metrics.go | 24 +- .../agent/v2/src/plugins/metrics_sender.go | 4 +- .../agent/v2/src/plugins/metrics_throlling.go | 82 ++++--- .../agent/sdk/v2/client/metric_reporter.go | 2 +- .../nginx/agent/sdk/v2/proto/metrics.pb.go | 76 +++--- .../nginx/agent/sdk/v2/proto/metrics.proto | 4 + 96 files changed, 944 insertions(+), 693 deletions(-) diff --git a/docs/proto/proto.md b/docs/proto/proto.md index 16078d6d4..4f7382cdd 100644 --- a/docs/proto/proto.md +++ b/docs/proto/proto.md @@ -1165,6 +1165,8 @@ Metric type enum | SYSTEM | 0 | System metric type | | INSTANCE | 1 | NGINX instance metric type | | AGENT | 2 | Agent metric type | +| CACHE_ZONE | 3 | Cache zone metric type | +| UPSTREAMS | 4 | Upstreams metric type | diff --git a/sdk/client/metric_reporter.go b/sdk/client/metric_reporter.go index 06fe5c328..46e46326b 100644 --- a/sdk/client/metric_reporter.go +++ b/sdk/client/metric_reporter.go @@ -157,7 +157,7 @@ func (r *metricReporter) Send(ctx context.Context, message Message) error { return r.handleGrpcError("Metric Reporter Channel Send", err) } - log.Tracef("MetricReporter sent metrics report %v", report) + log.Tracef("MetricReporter sent metrics report [Type: %d] %+v", report.Type, report) return nil }) diff --git a/sdk/proto/metrics.pb.go b/sdk/proto/metrics.pb.go index ee8c82794..e6ad339ef 100644 --- a/sdk/proto/metrics.pb.go +++ b/sdk/proto/metrics.pb.go @@ -35,18 +35,26 @@ const ( MetricsReport_INSTANCE MetricsReport_Type = 1 // Agent metric type MetricsReport_AGENT MetricsReport_Type = 2 + // Cache zone metric type + MetricsReport_CACHE_ZONE MetricsReport_Type = 3 + // Upstreams metric type + MetricsReport_UPSTREAMS MetricsReport_Type = 4 ) var MetricsReport_Type_name = map[int32]string{ 0: "SYSTEM", 1: "INSTANCE", 2: "AGENT", + 3: "CACHE_ZONE", + 4: "UPSTREAMS", } var MetricsReport_Type_value = map[string]int32{ - "SYSTEM": 0, - "INSTANCE": 1, - "AGENT": 2, + "SYSTEM": 0, + "INSTANCE": 1, + "AGENT": 2, + "CACHE_ZONE": 3, + "UPSTREAMS": 4, } func (x MetricsReport_Type) String() string { @@ -318,36 +326,38 @@ func init() { func init() { proto.RegisterFile("metrics.proto", fileDescriptor_6039342a2ba47b72) } var fileDescriptor_6039342a2ba47b72 = []byte{ - // 460 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x52, 0x41, 0x6f, 0xd3, 0x30, - 0x14, 0xc6, 0x6d, 0x36, 0x9a, 0xd7, 0x76, 0x2a, 0x3e, 0x55, 0xd5, 0x68, 0xaa, 0x1e, 0x50, 0x11, - 0x92, 0x23, 0x15, 0x21, 0x10, 0x88, 0xc3, 0xc2, 0x2a, 0xc4, 0xa4, 0xf4, 0xe0, 0xe4, 0x32, 0x6e, - 0x5e, 0xeb, 0x85, 0x68, 0x75, 0x1c, 0xd5, 0xee, 0x44, 0x6f, 0xfc, 0x3c, 0x8e, 0xfc, 0x82, 0x0a, - 0xf5, 0xd8, 0xff, 0x80, 0x84, 0x62, 0xa7, 0x5b, 0x27, 0xc2, 0x61, 0x97, 0x67, 0x3f, 0xeb, 0x7d, - 0xdf, 0x7b, 0xef, 0xfb, 0x0c, 0x6d, 0xc1, 0xf5, 0x32, 0x9d, 0x29, 0x92, 0x2f, 0xa5, 0x96, 0x18, - 0x5f, 0xbf, 0x21, 0x59, 0x92, 0x66, 0xdf, 0x09, 0x4b, 0x78, 0xa6, 0x89, 0x9a, 0xdf, 0xf4, 0x5a, - 0x33, 0x29, 0x84, 0xcc, 0x6c, 0x45, 0x0f, 0x12, 0x99, 0xc8, 0xf2, 0xee, 0x25, 0x52, 0x26, 0x0b, - 0xee, 0x9b, 0xec, 0x6a, 0x75, 0xed, 0xeb, 0x54, 0x70, 0xa5, 0x99, 0xc8, 0x6d, 0xc1, 0xf0, 0x0f, - 0x82, 0x76, 0x68, 0x1b, 0x50, 0x9e, 0xcb, 0xa5, 0xc6, 0xef, 0xc1, 0x11, 0x5c, 0xb3, 0x2e, 0x1a, - 0xa0, 0x51, 0x73, 0x7c, 0x4a, 0xfe, 0xed, 0x47, 0x42, 0xae, 0xd9, 0x9c, 0x69, 0x16, 0x34, 0x76, - 0x1b, 0xcf, 0x54, 0x53, 0x13, 0xf1, 0x39, 0x38, 0x7a, 0x9d, 0xf3, 0x6e, 0x6d, 0x80, 0x46, 0x27, - 0xe3, 0x17, 0xff, 0xc1, 0xde, 0x37, 0x23, 0xf1, 0x3a, 0xe7, 0x96, 0xa5, 0xc0, 0x51, 0x13, 0xf1, - 0x47, 0x70, 0x0a, 0xf6, 0x6e, 0x7d, 0x50, 0x1f, 0x35, 0xc7, 0x5e, 0x15, 0x4b, 0xa4, 0x99, 0x56, - 0x93, 0x4c, 0xa7, 0x7a, 0x6d, 0xe1, 0x05, 0x80, 0x9a, 0x38, 0x7c, 0x05, 0x4e, 0x41, 0x8b, 0x01, - 0x8e, 0xa3, 0xcb, 0x28, 0x9e, 0x84, 0x9d, 0x27, 0xb8, 0x05, 0x8d, 0x2f, 0xd3, 0x28, 0x3e, 0x9b, - 0x7e, 0x9a, 0x74, 0x10, 0x76, 0xe1, 0xe8, 0xec, 0xf3, 0x64, 0x1a, 0x77, 0x6a, 0xc3, 0x10, 0x5a, - 0x51, 0x2a, 0xf2, 0x05, 0xb7, 0x73, 0xe1, 0x53, 0x70, 0x32, 0x26, 0xb8, 0xd9, 0xde, 0xb5, 0xd4, - 0x45, 0x4e, 0x4d, 0xc4, 0x1e, 0x1c, 0xdd, 0xb2, 0xc5, 0xca, 0x2e, 0x88, 0x02, 0x77, 0xb7, 0xf1, - 0xec, 0x03, 0xb5, 0xc7, 0xf0, 0x02, 0xdc, 0xf3, 0x54, 0xf0, 0x4c, 0xa5, 0x32, 0x7b, 0x0c, 0x97, - 0x5b, 0xc1, 0xf5, 0xa3, 0x06, 0xcd, 0x83, 0x3d, 0xf1, 0x3b, 0x70, 0xef, 0xdc, 0x2b, 0xdd, 0xe9, - 0x11, 0xeb, 0x2f, 0xd9, 0xfb, 0x4b, 0xe2, 0x7d, 0x05, 0xbd, 0x2f, 0xc6, 0x21, 0xc0, 0x7c, 0x3f, - 0x95, 0x2a, 0x65, 0x7d, 0x5e, 0x25, 0xeb, 0xdd, 0xec, 0xc1, 0xc9, 0x6e, 0xe3, 0x1d, 0x80, 0xe8, - 0xc1, 0x1d, 0x5f, 0x42, 0x5b, 0x19, 0xcd, 0xca, 0x9f, 0xd9, 0x75, 0x0c, 0xe3, 0xa0, 0xd2, 0xa8, - 0x03, 0x71, 0x83, 0x67, 0xbb, 0x8d, 0xf7, 0x10, 0x4a, 0x1f, 0xa6, 0x17, 0x4e, 0xa3, 0xd6, 0xa9, - 0xd3, 0xa7, 0x65, 0x1a, 0xbc, 0xfd, 0xb9, 0xed, 0xa3, 0x5f, 0xdb, 0x3e, 0xfa, 0xbd, 0xed, 0xa3, - 0xaf, 0x2f, 0x93, 0x54, 0x7f, 0x5b, 0x5d, 0x91, 0x99, 0x14, 0xbe, 0xe9, 0xe3, 0x9b, 0x3e, 0xbe, - 0x9a, 0xdf, 0xf8, 0xb7, 0x63, 0xfb, 0xc7, 0x3f, 0x58, 0x25, 0x8e, 0xcd, 0xf1, 0xfa, 0x6f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xe6, 0xcd, 0x1e, 0xb5, 0x3d, 0x03, 0x00, 0x00, + // 487 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xc5, 0x8e, 0x5b, 0xe2, 0xc9, 0x87, 0xcc, 0x9e, 0xa2, 0xa8, 0xc4, 0x51, 0x0e, 0x28, 0x5c, + 0xd6, 0x52, 0x10, 0x02, 0x81, 0x38, 0xc4, 0xa9, 0x05, 0x44, 0x72, 0x40, 0x6b, 0x73, 0x68, 0x2f, + 0xc8, 0x4d, 0xb6, 0xc6, 0x6a, 0xd6, 0x6b, 0xc5, 0x9b, 0x8a, 0xdc, 0xf8, 0x21, 0xfc, 0x20, 0x8e, + 0xfc, 0x82, 0x08, 0xe5, 0x98, 0x5f, 0x81, 0xbc, 0xeb, 0xb4, 0x89, 0x08, 0x87, 0x5e, 0x66, 0x77, + 0xac, 0x79, 0xef, 0x8d, 0xdf, 0xb3, 0xa1, 0xc1, 0xa8, 0x58, 0x24, 0xd3, 0x1c, 0x67, 0x0b, 0x2e, + 0x38, 0x42, 0xd7, 0x2f, 0x71, 0x1a, 0x27, 0xe9, 0x77, 0x1c, 0xc5, 0x34, 0x15, 0x38, 0x9f, 0xdd, + 0xb4, 0xeb, 0x53, 0xce, 0x18, 0x4f, 0xd5, 0x44, 0x1b, 0x62, 0x1e, 0xf3, 0xf2, 0x6e, 0xc7, 0x9c, + 0xc7, 0x73, 0xea, 0xc8, 0xee, 0x6a, 0x79, 0xed, 0x88, 0x84, 0xd1, 0x5c, 0x44, 0x2c, 0x53, 0x03, + 0xbd, 0x9f, 0x3a, 0x34, 0x7c, 0x25, 0x40, 0x68, 0xc6, 0x17, 0x02, 0xbd, 0x01, 0x83, 0x51, 0x11, + 0xb5, 0xb4, 0xae, 0xd6, 0xaf, 0x0d, 0xce, 0xf0, 0xbf, 0x7a, 0xd8, 0xa7, 0x22, 0x9a, 0x45, 0x22, + 0x72, 0xab, 0xdb, 0xb5, 0x2d, 0xa7, 0x89, 0xac, 0xe8, 0x1c, 0x0c, 0xb1, 0xca, 0x68, 0x4b, 0xef, + 0x6a, 0xfd, 0xe6, 0xe0, 0xd9, 0x7f, 0xb0, 0xf7, 0x62, 0x38, 0x5c, 0x65, 0x54, 0xb1, 0x14, 0x38, + 0x22, 0x2b, 0x7a, 0x07, 0x46, 0xc1, 0xde, 0xaa, 0x74, 0x2b, 0xfd, 0xda, 0xc0, 0x3e, 0xc6, 0x12, + 0x88, 0x48, 0xe4, 0x5e, 0x2a, 0x12, 0xb1, 0x52, 0xf0, 0x02, 0x40, 0x64, 0xed, 0x8d, 0xc1, 0x28, + 0x68, 0x11, 0xc0, 0x69, 0x70, 0x11, 0x84, 0x9e, 0x6f, 0x3d, 0x42, 0x75, 0xa8, 0x7e, 0x9c, 0x04, + 0xe1, 0x70, 0x32, 0xf2, 0x2c, 0x0d, 0x99, 0x70, 0x32, 0x7c, 0xef, 0x4d, 0x42, 0x4b, 0x47, 0x4d, + 0x80, 0xd1, 0x70, 0xf4, 0xc1, 0xfb, 0x7a, 0xf9, 0x69, 0xe2, 0x59, 0x15, 0xd4, 0x00, 0xf3, 0xcb, + 0xe7, 0x20, 0x24, 0xde, 0xd0, 0x0f, 0x2c, 0xa3, 0xe7, 0x43, 0x3d, 0x48, 0x58, 0x36, 0xa7, 0x6a, + 0x6d, 0x74, 0x06, 0x46, 0x1a, 0x31, 0x2a, 0xcd, 0x31, 0x95, 0x72, 0xd1, 0x13, 0x59, 0x91, 0x0d, + 0x27, 0xb7, 0xd1, 0x7c, 0xa9, 0xde, 0x5f, 0x73, 0xcd, 0xed, 0xda, 0x56, 0x0f, 0x88, 0x3a, 0x7a, + 0x63, 0x30, 0xcf, 0x13, 0x46, 0xd3, 0x3c, 0xe1, 0xe9, 0x43, 0xb8, 0xcc, 0x23, 0x5c, 0x3f, 0x74, + 0xa8, 0xed, 0xd9, 0x80, 0x5e, 0x83, 0x79, 0x17, 0x6e, 0x19, 0x5e, 0x1b, 0xab, 0xf8, 0xf1, 0x2e, + 0x7e, 0x1c, 0xee, 0x26, 0xc8, 0xfd, 0x30, 0xf2, 0x01, 0x66, 0xbb, 0xad, 0xf2, 0xd2, 0xf5, 0xa7, + 0xc7, 0x5c, 0xbf, 0xdb, 0xdd, 0x6d, 0x6e, 0xd7, 0xf6, 0x1e, 0x88, 0xec, 0xdd, 0xd1, 0x05, 0x34, + 0x72, 0xe9, 0x59, 0xf9, 0xe1, 0xb6, 0x0c, 0xc9, 0xd8, 0x3d, 0x9a, 0xe3, 0x9e, 0xb9, 0xee, 0x93, + 0xed, 0xda, 0x3e, 0x84, 0x92, 0xc3, 0x76, 0x6c, 0x54, 0x75, 0xab, 0x42, 0x1e, 0x97, 0xad, 0xfb, + 0xea, 0xd7, 0xa6, 0xa3, 0xfd, 0xde, 0x74, 0xb4, 0x3f, 0x9b, 0x8e, 0x76, 0xf9, 0x3c, 0x4e, 0xc4, + 0xb7, 0xe5, 0x15, 0x9e, 0x72, 0xe6, 0x48, 0x1d, 0x47, 0xea, 0x38, 0xf9, 0xec, 0xc6, 0xb9, 0x1d, + 0xa8, 0x5f, 0xe0, 0xad, 0x72, 0xe2, 0x54, 0x1e, 0x2f, 0xfe, 0x06, 0x00, 0x00, 0xff, 0xff, 0x87, + 0xa1, 0xae, 0x17, 0x5c, 0x03, 0x00, 0x00, } func (m *MetricsReport) Marshal() (dAtA []byte, err error) { diff --git a/sdk/proto/metrics.proto b/sdk/proto/metrics.proto index e4cba46fd..fa3cf941d 100644 --- a/sdk/proto/metrics.proto +++ b/sdk/proto/metrics.proto @@ -17,6 +17,10 @@ message MetricsReport { INSTANCE = 1; // Agent metric type AGENT = 2; + // Cache zone metric type + CACHE_ZONE = 3; + // Upstreams metric type + UPSTREAMS = 4; } // Provides meta information about the metrics Metadata meta = 1 [(gogoproto.jsontag) = "meta"]; diff --git a/src/core/config/defaults.go b/src/core/config/defaults.go index 5450c6e41..194a6deb0 100644 --- a/src/core/config/defaults.go +++ b/src/core/config/defaults.go @@ -71,7 +71,7 @@ var ( BulkSize: 20, ReportInterval: 1 * time.Minute, CollectionInterval: 15 * time.Second, - Mode: "aggregation", + Mode: "aggregated", }, Features: agent_config.GetDefaultFeatures(), } diff --git a/src/core/metrics/aggregate.go b/src/core/metrics/aggregate.go index 8bbff11c4..38a129749 100644 --- a/src/core/metrics/aggregate.go +++ b/src/core/metrics/aggregate.go @@ -40,7 +40,7 @@ func dimChecksum(stats *proto.StatsEntity) string { // SaveCollections loops through one or more reports and get all the raw metrics for the Collections // Note this function operate on the Collections struct data directly. func SaveCollections(metricsCollections Collections, reports ...*proto.MetricsReport) Collections { - // could be multiple reports + // could be multiple reports, however they must all be of the same type. for _, report := range reports { metricsCollections.Count++ for _, stats := range report.GetData() { @@ -65,7 +65,7 @@ func SaveCollections(metricsCollections Collections, reports ...*proto.MetricsRe return metricsCollections } -func GenerateMetricsReport(metricsCollections Collections) *proto.MetricsReport { +func GenerateMetrics(metricsCollections Collections) []*proto.StatsEntity { results := make([]*proto.StatsEntity, 0, 200) @@ -77,11 +77,7 @@ func GenerateMetricsReport(metricsCollections Collections) *proto.MetricsReport )) } - return &proto.MetricsReport{ - Meta: &proto.Metadata{}, - Type: 0, - Data: results, - } + return results } func getAggregatedSimpleMetric(count int, internalMap map[string]float64) (simpleMetrics []*proto.SimpleMetric) { diff --git a/src/core/metrics/aggregate_test.go b/src/core/metrics/aggregate_test.go index 7275961b1..a09b35ac6 100644 --- a/src/core/metrics/aggregate_test.go +++ b/src/core/metrics/aggregate_test.go @@ -145,11 +145,11 @@ func TestGenerateAggregationReport(t *testing.T) { metricsCollections.Data[csum].RunningSumMap["system.cpu.system"] = 200.2 metricsCollections.Data[csum].RunningSumMap["system.undefined_method"] = 1000 - report := GenerateMetricsReport(metricsCollections) - log.Info(report) + results := GenerateMetrics(metricsCollections) + log.Info(results) - assert.NotNil(t, report) - for _, stats := range report.GetData() { + assert.NotEmpty(t, results) + for _, stats := range results { simplemetrics := stats.GetSimplemetrics() for _, v := range simplemetrics { switch { diff --git a/src/core/metrics/collectors/container.go b/src/core/metrics/collectors/container.go index f01550a33..1beb937f7 100644 --- a/src/core/metrics/collectors/container.go +++ b/src/core/metrics/collectors/container.go @@ -13,7 +13,6 @@ import ( log "github.com/sirupsen/logrus" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" @@ -27,7 +26,7 @@ var ( type ContainerCollector struct { sources []metrics.Source - buf chan *proto.StatsEntity + buf chan *metrics.StatsEntityWrapper dim *metrics.CommonDim env core.Environment } @@ -42,7 +41,7 @@ func NewContainerCollector(env core.Environment, conf *config.Config) *Container return &ContainerCollector{ sources: containerSources, - buf: make(chan *proto.StatsEntity, 65535), + buf: make(chan *metrics.StatsEntityWrapper, 65535), dim: metrics.NewCommonDim(env.NewHostInfo("agentVersion", &conf.Tags, conf.ConfigDirs, false), conf, ""), env: env, } @@ -59,7 +58,7 @@ func (c *ContainerCollector) collectMetrics(ctx context.Context) { wg.Wait() } -func (c *ContainerCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *ContainerCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.collectMetrics(ctx) @@ -69,7 +68,7 @@ func (c *ContainerCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m case <-ctx.Done(): return case sample := <-c.buf: - sample.Dimensions = append(commonDims, sample.Dimensions...) + sample.Data.Dimensions = append(commonDims, sample.Data.Dimensions...) select { case <-ctx.Done(): diff --git a/src/core/metrics/collectors/container_test.go b/src/core/metrics/collectors/container_test.go index 7951be844..bb8cc9597 100644 --- a/src/core/metrics/collectors/container_test.go +++ b/src/core/metrics/collectors/container_test.go @@ -57,7 +57,7 @@ func TestContainerCollector_Collect(t *testing.T) { mockSource1, mockSource2, }, - buf: make(chan *proto.StatsEntity), + buf: make(chan *metrics.StatsEntityWrapper), dim: &metrics.CommonDim{}, } @@ -65,10 +65,10 @@ func TestContainerCollector_Collect(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(1) - channel := make(chan *proto.StatsEntity) + channel := make(chan *metrics.StatsEntityWrapper) go containerCollector.Collect(ctx, wg, channel) - containerCollector.buf <- &proto.StatsEntity{Dimensions: []*proto.Dimension{{Name: "new_dim", Value: "123"}}} + containerCollector.buf <- &metrics.StatsEntityWrapper{Type: proto.MetricsReport_SYSTEM, Data: &proto.StatsEntity{Dimensions: []*proto.Dimension{{Name: "new_dim", Value: "123"}}}} actual := <-channel mockSource1.AssertExpectations(t) @@ -83,7 +83,7 @@ func TestContainerCollector_Collect(t *testing.T) { {Name: "nginx_id", Value: ""}, {Name: "new_dim", Value: "123"}, } - assert.Equal(t, expectedDimensions, actual.Dimensions) + assert.Equal(t, expectedDimensions, actual.Data.Dimensions) } func TestContainerCollector_UpdateConfig(t *testing.T) { diff --git a/src/core/metrics/collectors/nginx.go b/src/core/metrics/collectors/nginx.go index 11f08ae39..93a3539ab 100644 --- a/src/core/metrics/collectors/nginx.go +++ b/src/core/metrics/collectors/nginx.go @@ -11,7 +11,6 @@ import ( "context" "sync" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" @@ -25,7 +24,7 @@ var ( type NginxCollector struct { sources []metrics.NginxSource - buf chan *proto.StatsEntity + buf chan *metrics.StatsEntityWrapper dimensions *metrics.CommonDim collectorConf *metrics.NginxCollectorConfig env core.Environment @@ -40,7 +39,7 @@ func NewNginxCollector(conf *config.Config, env core.Environment, collectorConf return &NginxCollector{ sources: buildSources(dimensions, binary, collectorConf), - buf: make(chan *proto.StatsEntity, 65535), + buf: make(chan *metrics.StatsEntityWrapper, 65535), dimensions: dimensions, collectorConf: collectorConf, env: env, @@ -81,7 +80,7 @@ func (c *NginxCollector) collectMetrics(ctx context.Context) { wg.Wait() } -func (c *NginxCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.collectMetrics(ctx) for { diff --git a/src/core/metrics/collectors/nginx_test.go b/src/core/metrics/collectors/nginx_test.go index 552adf778..72372fc31 100644 --- a/src/core/metrics/collectors/nginx_test.go +++ b/src/core/metrics/collectors/nginx_test.go @@ -169,7 +169,7 @@ func TestNginxCollector_Collect(t *testing.T) { ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) - go nginxCollector.Collect(ctx, wg, make(chan<- *proto.StatsEntity)) + go nginxCollector.Collect(ctx, wg, make(chan<- *metrics.StatsEntityWrapper)) time.Sleep(10 * time.Millisecond) mockNginxSource1.AssertExpectations(t) diff --git a/src/core/metrics/collectors/source_mocks_test.go b/src/core/metrics/collectors/source_mocks_test.go index d61fe2da3..b6fa5a5ad 100644 --- a/src/core/metrics/collectors/source_mocks_test.go +++ b/src/core/metrics/collectors/source_mocks_test.go @@ -11,7 +11,6 @@ import ( "context" "sync" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core/metrics" "github.com/stretchr/testify/mock" ) @@ -26,7 +25,7 @@ type NginxSourceMock struct { mock.Mock } -func (m *NginxSourceMock) Collect(ctx context.Context, wg *sync.WaitGroup, statsChannel chan<- *proto.StatsEntity) { +func (m *NginxSourceMock) Collect(ctx context.Context, wg *sync.WaitGroup, statsChannel chan<- *metrics.StatsEntityWrapper) { m.Called(ctx, wg, statsChannel) wg.Done() } @@ -43,7 +42,7 @@ type SourceMock struct { mock.Mock } -func (m *SourceMock) Collect(ctx context.Context, wg *sync.WaitGroup, statsChannel chan<- *proto.StatsEntity) { +func (m *SourceMock) Collect(ctx context.Context, wg *sync.WaitGroup, statsChannel chan<- *metrics.StatsEntityWrapper) { m.Called(ctx, wg, statsChannel) wg.Done() } diff --git a/src/core/metrics/collectors/system.go b/src/core/metrics/collectors/system.go index b4bf7a2d8..3fcb054d4 100644 --- a/src/core/metrics/collectors/system.go +++ b/src/core/metrics/collectors/system.go @@ -11,7 +11,6 @@ import ( "context" "sync" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" @@ -24,7 +23,7 @@ var ( type SystemCollector struct { sources []metrics.Source - buf chan *proto.StatsEntity + buf chan *metrics.StatsEntityWrapper dim *metrics.CommonDim env core.Environment } @@ -53,7 +52,7 @@ func NewSystemCollector(env core.Environment, conf *config.Config) *SystemCollec return &SystemCollector{ sources: systemSources, - buf: make(chan *proto.StatsEntity, 65535), + buf: make(chan *metrics.StatsEntityWrapper, 65535), dim: metrics.NewCommonDim(env.NewHostInfo("agentVersion", &conf.Tags, conf.ConfigDirs, false), conf, ""), env: env, } @@ -70,7 +69,7 @@ func (c *SystemCollector) collectMetrics(ctx context.Context) { wg.Wait() } -func (c *SystemCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *SystemCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.collectMetrics(ctx) @@ -80,7 +79,7 @@ func (c *SystemCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m cha case <-ctx.Done(): return case sample := <-c.buf: - sample.Dimensions = append(commonDims, sample.Dimensions...) + sample.Data.Dimensions = append(commonDims, sample.Data.Dimensions...) select { case <-ctx.Done(): diff --git a/src/core/metrics/collectors/system_test.go b/src/core/metrics/collectors/system_test.go index 9462d9439..a7cd9bdeb 100644 --- a/src/core/metrics/collectors/system_test.go +++ b/src/core/metrics/collectors/system_test.go @@ -89,7 +89,7 @@ func TestSystemCollector_Collect(t *testing.T) { mockSource1, mockSource2, }, - buf: make(chan *proto.StatsEntity), + buf: make(chan *metrics.StatsEntityWrapper), dim: &metrics.CommonDim{}, } @@ -97,10 +97,10 @@ func TestSystemCollector_Collect(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(1) - channel := make(chan *proto.StatsEntity) + channel := make(chan *metrics.StatsEntityWrapper) go systemCollector.Collect(ctx, wg, channel) - systemCollector.buf <- &proto.StatsEntity{Dimensions: []*proto.Dimension{{Name: "new_dim", Value: "123"}}} + systemCollector.buf <- &metrics.StatsEntityWrapper{Type: proto.MetricsReport_SYSTEM, Data: &proto.StatsEntity{Dimensions: []*proto.Dimension{{Name: "new_dim", Value: "123"}}}} actual := <-channel mockSource1.AssertExpectations(t) @@ -115,7 +115,7 @@ func TestSystemCollector_Collect(t *testing.T) { {Name: "nginx_id", Value: ""}, {Name: "new_dim", Value: "123"}, } - assert.Equal(t, expectedDimensions, actual.Dimensions) + assert.Equal(t, expectedDimensions, actual.Data.Dimensions) } func TestSystemCollector_UpdateConfig(t *testing.T) { diff --git a/src/core/metrics/metrics_util.go b/src/core/metrics/metrics_util.go index 1d5823b72..2cae8caf0 100644 --- a/src/core/metrics/metrics_util.go +++ b/src/core/metrics/metrics_util.go @@ -15,23 +15,34 @@ import ( "time" "github.com/nginx/agent/sdk/v2/proto" + "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/gogo/protobuf/types" ) type Collector interface { - Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) + Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *StatsEntityWrapper) UpdateConfig(config *config.Config) } type Source interface { - Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) + Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *StatsEntityWrapper) } type NginxSource interface { - Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) + Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *StatsEntityWrapper) Update(dimensions *CommonDim, collectorConf *NginxCollectorConfig) Stop() } + +type StatsEntityWrapper struct { + Type proto.MetricsReport_Type + Data *proto.StatsEntity +} + +type MetricsReportBundle struct { + Data []*proto.MetricsReport +} + type NginxCollectorConfig struct { NginxId string StubStatus string @@ -44,6 +55,14 @@ type NginxCollectorConfig struct { ClientVersion int } +func NewStatsEntityWrapper(dims []*proto.Dimension, samples []*proto.SimpleMetric, seType proto.MetricsReport_Type) *StatsEntityWrapper { + return &StatsEntityWrapper{seType, &proto.StatsEntity{ + Timestamp: types.TimestampNow(), + Dimensions: dims, + Simplemetrics: samples, + }} +} + func NewStatsEntity(dims []*proto.Dimension, samples []*proto.SimpleMetric) *proto.StatsEntity { return &proto.StatsEntity{ Timestamp: types.TimestampNow(), @@ -366,3 +385,29 @@ func GetCalculationMap() map[string]string { "container.mem.oom.kill": "avg", } } + +func GenerateMetricsReportBundle(entities []*StatsEntityWrapper) core.Payload { + reportMap := make(map[proto.MetricsReport_Type]*proto.MetricsReport, 0) + + for _, entity := range entities { + if _, ok := reportMap[entity.Type]; !ok { + reportMap[entity.Type] = &proto.MetricsReport{ + Meta: &proto.Metadata{ + Timestamp: types.TimestampNow(), + }, + Type: entity.Type, + Data: make([]*proto.StatsEntity, 0), + } + } + reportMap[entity.Type].Data = append(reportMap[entity.Type].Data, entity.Data) + } + + if len(reportMap) > 0 { + bundle := &MetricsReportBundle{Data: []*proto.MetricsReport{}} + for _, report := range reportMap { + bundle.Data = append(bundle.Data, report) + } + return bundle + } + return nil +} diff --git a/src/core/metrics/sources/common.go b/src/core/metrics/sources/common.go index 8a1fa3fab..3041ee512 100644 --- a/src/core/metrics/sources/common.go +++ b/src/core/metrics/sources/common.go @@ -88,11 +88,11 @@ func Delta(current, previous map[string]map[string]float64) map[string]map[strin return diff } -func SendNginxDownStatus(ctx context.Context, dims []*proto.Dimension, m chan<- *proto.StatsEntity) { +func SendNginxDownStatus(ctx context.Context, dims []*proto.Dimension, m chan<- *metrics.StatsEntityWrapper) { simpleMetrics := []*proto.SimpleMetric{newFloatMetric("nginx.status", float64(0))} select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity(dims, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE): } } diff --git a/src/core/metrics/sources/common_test.go b/src/core/metrics/sources/common_test.go index fa63717a4..5d55917b5 100644 --- a/src/core/metrics/sources/common_test.go +++ b/src/core/metrics/sources/common_test.go @@ -9,6 +9,7 @@ package sources import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "sort" "testing" @@ -180,10 +181,10 @@ func TestCommonDelta(t *testing.T) { } func TestCommonSendNginxDownStatus(t *testing.T) { - m := make(chan *proto.StatsEntity, 1) + m := make(chan *metrics.StatsEntityWrapper, 1) expected := &proto.SimpleMetric{Name: "nginx.status", Value: 0.0} SendNginxDownStatus(context.TODO(), []*proto.Dimension{}, m) actual := <-m - assert.Equal(t, 1, len(actual.Simplemetrics)) - assert.Equal(t, expected, actual.Simplemetrics[0]) + assert.Equal(t, 1, len(actual.Data.Simplemetrics)) + assert.Equal(t, expected, actual.Data.Simplemetrics[0]) } diff --git a/src/core/metrics/sources/container_cpu.go b/src/core/metrics/sources/container_cpu.go index 6023d2aa3..474f6871c 100644 --- a/src/core/metrics/sources/container_cpu.go +++ b/src/core/metrics/sources/container_cpu.go @@ -47,7 +47,7 @@ func NewContainerCPUSource(namespace string, basePath string) *ContainerCPU { return &ContainerCPU{basePath, cgroup.IsCgroupV2(basePath), NewMetricSourceLogger(), &namedMetric{namespace, CpuGroup}} } -func (c *ContainerCPU) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *ContainerCPU) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { log.Trace("Collecting container CPU metrics") defer wg.Done() @@ -205,7 +205,7 @@ func (c *ContainerCPU) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/src/core/metrics/sources/container_cpu_test.go b/src/core/metrics/sources/container_cpu_test.go index cd8c30260..948754390 100644 --- a/src/core/metrics/sources/container_cpu_test.go +++ b/src/core/metrics/sources/container_cpu_test.go @@ -9,6 +9,7 @@ package sources import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "path" "runtime" "sort" @@ -116,7 +117,7 @@ func TestContainerCPUSource(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - actual := make(chan *proto.StatsEntity, 1) + actual := make(chan *metrics.StatsEntityWrapper, 1) ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) @@ -130,10 +131,10 @@ func TestContainerCPUSource(t *testing.T) { sort.SliceStable(test.stats.Simplemetrics, func(i, j int) bool { return test.stats.Simplemetrics[i].Name < test.stats.Simplemetrics[j].Name }) - sort.SliceStable(result.Simplemetrics, func(i, j int) bool { - return result.Simplemetrics[i].Name < result.Simplemetrics[j].Name + sort.SliceStable(result.Data.Simplemetrics, func(i, j int) bool { + return result.Data.Simplemetrics[i].Name < result.Data.Simplemetrics[j].Name }) - assert.Equal(tt, test.stats.Simplemetrics, result.Simplemetrics) + assert.Equal(tt, test.stats.Simplemetrics, result.Data.Simplemetrics) case <-time.After(10 * time.Millisecond): assert.Nil(tt, test.stats) } diff --git a/src/core/metrics/sources/container_mem.go b/src/core/metrics/sources/container_mem.go index 20b08f109..01d0bcd7c 100644 --- a/src/core/metrics/sources/container_mem.go +++ b/src/core/metrics/sources/container_mem.go @@ -40,7 +40,7 @@ func NewContainerMemorySource(namespace string, basePath string) *ContainerMemor return &ContainerMemory{basePath, cgroup.IsCgroupV2(basePath), NewMetricSourceLogger(), &namedMetric{namespace, MemoryGroup}} } -func (c *ContainerMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *ContainerMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { log.Trace("Collecting container memory metrics") defer wg.Done() @@ -70,7 +70,7 @@ func (c *ContainerMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m cha select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/src/core/metrics/sources/container_mem_test.go b/src/core/metrics/sources/container_mem_test.go index 5ba1f5202..d7ca4eb76 100644 --- a/src/core/metrics/sources/container_mem_test.go +++ b/src/core/metrics/sources/container_mem_test.go @@ -9,6 +9,7 @@ package sources import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "path" "runtime" "sort" @@ -60,7 +61,7 @@ func TestContainerMemorySource(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - actual := make(chan *proto.StatsEntity, 1) + actual := make(chan *metrics.StatsEntityWrapper, 1) ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) @@ -74,10 +75,10 @@ func TestContainerMemorySource(t *testing.T) { sort.SliceStable(test.stats.Simplemetrics, func(i, j int) bool { return test.stats.Simplemetrics[i].Name < test.stats.Simplemetrics[j].Name }) - sort.SliceStable(result.Simplemetrics, func(i, j int) bool { - return result.Simplemetrics[i].Name < result.Simplemetrics[j].Name + sort.SliceStable(result.Data.Simplemetrics, func(i, j int) bool { + return result.Data.Simplemetrics[i].Name < result.Data.Simplemetrics[j].Name }) - assert.Equal(tt, test.stats.Simplemetrics, result.Simplemetrics) + assert.Equal(tt, test.stats.Simplemetrics, result.Data.Simplemetrics) case <-time.After(10 * time.Millisecond): assert.Nil(tt, test.stats) } diff --git a/src/core/metrics/sources/cpu.go b/src/core/metrics/sources/cpu.go index 4e1a8eaa2..a5053441d 100644 --- a/src/core/metrics/sources/cpu.go +++ b/src/core/metrics/sources/cpu.go @@ -69,7 +69,7 @@ func diffTimeStat(t1, t2 cpu.TimesStat) cpu.TimesStat { } } -func (c *CPUTimes) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *CPUTimes) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() var simpleMetrics []*proto.SimpleMetric if c.isDocker { @@ -126,6 +126,6 @@ func (c *CPUTimes) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *pr select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/src/core/metrics/sources/cpu_test.go b/src/core/metrics/sources/cpu_test.go index ebdefec1b..06ff40e3c 100644 --- a/src/core/metrics/sources/cpu_test.go +++ b/src/core/metrics/sources/cpu_test.go @@ -9,13 +9,13 @@ package sources import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "path" "runtime" "sort" "sync" "testing" - "github.com/nginx/agent/sdk/v2/proto" cgroup "github.com/nginx/agent/v2/src/core/metrics/sources/cgroup" tutils "github.com/nginx/agent/v2/test/utils" @@ -108,14 +108,14 @@ func TestCPUTimesCollect_VM(t *testing.T) { ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) - channel := make(chan *proto.StatsEntity, 1) + channel := make(chan *metrics.StatsEntityWrapper, 1) cpuTimes.Collect(ctx, wg, channel) wg.Wait() actual := <-channel actualMetricNames := []string{} - for _, simpleMetric := range actual.Simplemetrics { + for _, simpleMetric := range actual.Data.Simplemetrics { actualMetricNames = append(actualMetricNames, simpleMetric.Name) switch simpleMetric.Name { case "test.cpu.idle": @@ -151,14 +151,14 @@ func TestCPUTimesCollect_VM(t *testing.T) { ctx = context.TODO() wg = &sync.WaitGroup{} wg.Add(1) - channel = make(chan *proto.StatsEntity, 1) + channel = make(chan *metrics.StatsEntityWrapper, 1) cpuTimes.Collect(ctx, wg, channel) wg.Wait() actual = <-channel actualMetricNames = []string{} - for _, simpleMetric := range actual.Simplemetrics { + for _, simpleMetric := range actual.Data.Simplemetrics { actualMetricNames = append(actualMetricNames, simpleMetric.Name) switch simpleMetric.Name { case "test.cpu.idle": @@ -194,13 +194,13 @@ func TestCPUTimesCollect_Container(t *testing.T) { ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) - channel := make(chan *proto.StatsEntity, 1) + channel := make(chan *metrics.StatsEntityWrapper, 1) cpuTimes.Collect(ctx, wg, channel) wg.Wait() actual := <-channel actualMetricNames := []string{} - for _, simpleMetric := range actual.Simplemetrics { + for _, simpleMetric := range actual.Data.Simplemetrics { actualMetricNames = append(actualMetricNames, simpleMetric.Name) } sort.Strings(actualMetricNames) diff --git a/src/core/metrics/sources/disk.go b/src/core/metrics/sources/disk.go index 6de927d32..909ee4a85 100644 --- a/src/core/metrics/sources/disk.go +++ b/src/core/metrics/sources/disk.go @@ -30,7 +30,7 @@ func NewDiskSource(namespace string) *Disk { return &Disk{NewMetricSourceLogger(), &namedMetric{namespace, "disk"}, disks} } -func (c *Disk) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *Disk) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() for _, part := range c.disks { if part.Device == "" || part.Fstype == "" { @@ -54,7 +54,7 @@ func (c *Disk) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto. case <-ctx.Done(): return // mount point is not a common dim - case m <- metrics.NewStatsEntity([]*proto.Dimension{{Name: MOUNT_POINT, Value: part.Mountpoint}}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{{Name: MOUNT_POINT, Value: part.Mountpoint}}, simpleMetrics, proto.MetricsReport_SYSTEM): } } } diff --git a/src/core/metrics/sources/disk_io.go b/src/core/metrics/sources/disk_io.go index 30b13ea39..eaa961cd2 100644 --- a/src/core/metrics/sources/disk_io.go +++ b/src/core/metrics/sources/disk_io.go @@ -37,7 +37,7 @@ func NewDiskIOSource(namespace string, env core.Environment) *DiskIO { return &DiskIO{namedMetric: &namedMetric{namespace, "io"}, env: env, diskIOStatsFunc: disk.IOCountersWithContext} } -func (dio *DiskIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (dio *DiskIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() dio.init.Do(func() { dio.diskDevs, _ = dio.env.DiskDevices() @@ -60,7 +60,7 @@ func (dio *DiskIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *pr // the Controller 3.x was labelling it wrong. However, changing this on the Analytics side // would involve a lot of changes (UI, API, Schema and Ingestion). So we are using mount_point // dimension for now. - case m <- metrics.NewStatsEntity([]*proto.Dimension{{Name: MOUNT_POINT, Value: k}}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{{Name: MOUNT_POINT, Value: k}}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/src/core/metrics/sources/disk_io_test.go b/src/core/metrics/sources/disk_io_test.go index 37452c5e8..d6a3fa967 100644 --- a/src/core/metrics/sources/disk_io_test.go +++ b/src/core/metrics/sources/disk_io_test.go @@ -9,11 +9,11 @@ package sources import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "sort" "sync" "testing" - "github.com/nginx/agent/sdk/v2/proto" tutils "github.com/nginx/agent/v2/test/utils" "github.com/shirou/gopsutil/v3/disk" "github.com/stretchr/testify/assert" @@ -40,14 +40,14 @@ func TestDiskIOCollect(t *testing.T) { ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) - channel := make(chan *proto.StatsEntity, 100) + channel := make(chan *metrics.StatsEntityWrapper, 100) diskio.Collect(ctx, wg, channel) wg.Wait() actual := <-channel actualMetricNames := []string{} - for _, simpleMetric := range actual.Simplemetrics { + for _, simpleMetric := range actual.Data.Simplemetrics { actualMetricNames = append(actualMetricNames, simpleMetric.Name) } sort.Strings(actualMetricNames) diff --git a/src/core/metrics/sources/disk_test.go b/src/core/metrics/sources/disk_test.go index dbc0d0fbe..ac5f552b3 100644 --- a/src/core/metrics/sources/disk_test.go +++ b/src/core/metrics/sources/disk_test.go @@ -9,11 +9,11 @@ package sources import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "sort" "sync" "testing" - "github.com/nginx/agent/sdk/v2/proto" "github.com/stretchr/testify/assert" ) @@ -33,14 +33,14 @@ func TestDiskCollect(t *testing.T) { ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) - channel := make(chan *proto.StatsEntity, 100) + channel := make(chan *metrics.StatsEntityWrapper, 100) disk.Collect(ctx, wg, channel) wg.Wait() actual := <-channel actualMetricNames := []string{} - for _, simpleMetric := range actual.Simplemetrics { + for _, simpleMetric := range actual.Data.Simplemetrics { actualMetricNames = append(actualMetricNames, simpleMetric.Name) } sort.Strings(actualMetricNames) diff --git a/src/core/metrics/sources/load.go b/src/core/metrics/sources/load.go index 29915ba4a..ca1e7ab7d 100644 --- a/src/core/metrics/sources/load.go +++ b/src/core/metrics/sources/load.go @@ -27,7 +27,7 @@ func NewLoadSource(namespace string) *Load { return &Load{logger: NewMetricSourceLogger(), namedMetric: &namedMetric{namespace, "load"}, avgStatsFunc: load.Avg} } -func (c *Load) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *Load) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() loadStats, err := c.avgStatsFunc() if err != nil { @@ -43,6 +43,6 @@ func (c *Load) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto. select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/src/core/metrics/sources/load_test.go b/src/core/metrics/sources/load_test.go index 5c76280d6..756b704f2 100644 --- a/src/core/metrics/sources/load_test.go +++ b/src/core/metrics/sources/load_test.go @@ -9,11 +9,11 @@ package sources import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "sort" "sync" "testing" - "github.com/nginx/agent/sdk/v2/proto" "github.com/shirou/gopsutil/v3/load" "github.com/stretchr/testify/assert" ) @@ -37,14 +37,14 @@ func TestLoadCollect(t *testing.T) { ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) - channel := make(chan *proto.StatsEntity, 100) + channel := make(chan *metrics.StatsEntityWrapper, 100) loadSource.Collect(ctx, wg, channel) wg.Wait() actual := <-channel actualMetricNames := []string{} - for _, simpleMetric := range actual.Simplemetrics { + for _, simpleMetric := range actual.Data.Simplemetrics { actualMetricNames = append(actualMetricNames, simpleMetric.Name) } sort.Strings(actualMetricNames) diff --git a/src/core/metrics/sources/mem.go b/src/core/metrics/sources/mem.go index c02d441f9..c83e724ca 100644 --- a/src/core/metrics/sources/mem.go +++ b/src/core/metrics/sources/mem.go @@ -38,7 +38,7 @@ func NewVirtualMemorySource(namespace string, env core.Environment) *VirtualMemo return &VirtualMemory{NewMetricSourceLogger(), &namedMetric{namespace, MemoryGroup}, statFunc} } -func (c *VirtualMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *VirtualMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() memstats, err := c.statFunc() if err != nil { @@ -66,6 +66,6 @@ func (c *VirtualMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m chan< select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/src/core/metrics/sources/mem_test.go b/src/core/metrics/sources/mem_test.go index b09b668a8..14a9c484c 100644 --- a/src/core/metrics/sources/mem_test.go +++ b/src/core/metrics/sources/mem_test.go @@ -9,11 +9,11 @@ package sources import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "sort" "sync" "testing" - "github.com/nginx/agent/sdk/v2/proto" cgroup "github.com/nginx/agent/v2/src/core/metrics/sources/cgroup" tutils "github.com/nginx/agent/v2/test/utils" "github.com/shirou/gopsutil/v3/mem" @@ -58,14 +58,14 @@ func TestVirtualMemoryCollect(t *testing.T) { ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) - channel := make(chan *proto.StatsEntity, 1) + channel := make(chan *metrics.StatsEntityWrapper, 1) go virtualMemorySource.Collect(ctx, wg, channel) wg.Wait() actual := <-channel actualMetricNames := []string{} - for _, simpleMetric := range actual.Simplemetrics { + for _, simpleMetric := range actual.Data.Simplemetrics { actualMetricNames = append(actualMetricNames, simpleMetric.Name) } sort.Strings(actualMetricNames) diff --git a/src/core/metrics/sources/net_io.go b/src/core/metrics/sources/net_io.go index 6e1b27b93..44bec29d8 100644 --- a/src/core/metrics/sources/net_io.go +++ b/src/core/metrics/sources/net_io.go @@ -46,7 +46,7 @@ func NewNetIOSource(namespace string, env core.Environment) *NetIO { } } -func (nio *NetIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (nio *NetIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() nio.init.Do(func() { ifs, err := nio.newNetInterfaces(ctx) @@ -87,7 +87,7 @@ func (nio *NetIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *pro case <-ctx.Done(): return // network_interface is not a common dim - case m <- metrics.NewStatsEntity([]*proto.Dimension{{Name: NETWORK_INTERFACE, Value: k}}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{{Name: NETWORK_INTERFACE, Value: k}}, simpleMetrics, proto.MetricsReport_SYSTEM): } } @@ -106,7 +106,7 @@ func (nio *NetIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *pro nio.netOverflows = overflows simpleMetrics := nio.convertSamplesToSimpleMetrics(totalStats) - m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics) + m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM) nio.netIOStats = currentNetIOStats } diff --git a/src/core/metrics/sources/net_io_test.go b/src/core/metrics/sources/net_io_test.go index 1f04949be..fa388fda1 100644 --- a/src/core/metrics/sources/net_io_test.go +++ b/src/core/metrics/sources/net_io_test.go @@ -9,11 +9,11 @@ package sources import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "sort" "sync" "testing" - "github.com/nginx/agent/sdk/v2/proto" tutils "github.com/nginx/agent/v2/test/utils" "github.com/shirou/gopsutil/v3/net" "github.com/stretchr/testify/assert" @@ -48,14 +48,14 @@ func TestNetIOCollect(t *testing.T) { ctx := context.TODO() wg := &sync.WaitGroup{} wg.Add(1) - channel := make(chan *proto.StatsEntity, 100) + channel := make(chan *metrics.StatsEntityWrapper, 100) nioSource.Collect(ctx, wg, channel) wg.Wait() actual := <-channel actualMetricNames := []string{} - for _, simpleMetric := range actual.Simplemetrics { + for _, simpleMetric := range actual.Data.Simplemetrics { actualMetricNames = append(actualMetricNames, simpleMetric.Name) } sort.Strings(actualMetricNames) diff --git a/src/core/metrics/sources/nginx_access_log.go b/src/core/metrics/sources/nginx_access_log.go index 4bcd71b79..fec3e7477 100644 --- a/src/core/metrics/sources/nginx_access_log.go +++ b/src/core/metrics/sources/nginx_access_log.go @@ -67,7 +67,7 @@ type NginxAccessLog struct { binary core.NginxBinary nginxType string collectionInterval time.Duration - buf []*proto.StatsEntity + buf []*metrics.StatsEntityWrapper logger *MetricSourceLogger } @@ -88,7 +88,7 @@ func NewNginxAccessLog( binary, nginxType, collectionInterval, - []*proto.StatsEntity{}, + []*metrics.StatsEntityWrapper{}, NewMetricSourceLogger(), } @@ -105,7 +105,7 @@ func NewNginxAccessLog( return nginxAccessLog } -func (c *NginxAccessLog) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxAccessLog) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.collectLogStats(ctx, m) } @@ -149,7 +149,7 @@ func (c *NginxAccessLog) Stop() { log.Debugf("Stopping NginxAccessLog source for nginx id: %v", c.baseDimensions.NginxId) } -func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *proto.StatsEntity) { +func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *metrics.StatsEntityWrapper) { c.mu.Lock() defer c.mu.Unlock() logs := c.binary.GetAccessLogs() @@ -178,7 +178,7 @@ func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *proto.St for _, stat := range c.buf { m <- stat } - c.buf = []*proto.StatsEntity{} + c.buf = []*metrics.StatsEntityWrapper{} } func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string) { @@ -337,7 +337,7 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string httpCounters, upstreamCounters, upstreamCacheCounters = getDefaultCounters() gzipRatios, requestLengths, requestTimes, upstreamResponseLength, upstreamResponseTimes, upstreamConnectTimes, upstreamHeaderTimes = []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{} - c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics)) + c.buf = append(c.buf, metrics.NewStatsEntityWrapper(c.baseDimensions.ToDimensions(), simpleMetrics, proto.MetricsReport_INSTANCE)) mu.Unlock() diff --git a/src/core/metrics/sources/nginx_access_log_test.go b/src/core/metrics/sources/nginx_access_log_test.go index e4b35a00e..faf1f839f 100644 --- a/src/core/metrics/sources/nginx_access_log_test.go +++ b/src/core/metrics/sources/nginx_access_log_test.go @@ -1593,11 +1593,11 @@ func TestAccessLogStats(t *testing.T) { sort.SliceStable(test.expectedStats.GetSimplemetrics(), func(i, j int) bool { return test.expectedStats.GetSimplemetrics()[i].Name < test.expectedStats.GetSimplemetrics()[j].Name }) - sort.SliceStable(nginxAccessLog.buf[0].GetSimplemetrics(), func(i, j int) bool { - return nginxAccessLog.buf[0].GetSimplemetrics()[i].Name < nginxAccessLog.buf[0].GetSimplemetrics()[j].Name + sort.SliceStable(nginxAccessLog.buf[0].Data.GetSimplemetrics(), func(i, j int) bool { + return nginxAccessLog.buf[0].Data.GetSimplemetrics()[i].Name < nginxAccessLog.buf[0].Data.GetSimplemetrics()[j].Name }) - assert.Equal(tt, test.expectedStats.GetSimplemetrics(), nginxAccessLog.buf[0].GetSimplemetrics()) + assert.Equal(tt, test.expectedStats.GetSimplemetrics(), nginxAccessLog.buf[0].Data.GetSimplemetrics()) }) } } diff --git a/src/core/metrics/sources/nginx_error_log.go b/src/core/metrics/sources/nginx_error_log.go index b9889ed59..1e8cd77ec 100644 --- a/src/core/metrics/sources/nginx_error_log.go +++ b/src/core/metrics/sources/nginx_error_log.go @@ -9,11 +9,11 @@ package sources import ( "context" + "github.com/nginx/agent/sdk/v2/proto" re "regexp" "sync" "time" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" "github.com/nginx/agent/v2/src/core/metrics/sources/tailer" @@ -66,7 +66,7 @@ type NginxErrorLog struct { binary core.NginxBinary nginxType string collectionInterval time.Duration - buf []*proto.StatsEntity + buf []*metrics.StatsEntityWrapper } func NewNginxErrorLog( @@ -85,7 +85,7 @@ func NewNginxErrorLog( binary, nginxType, collectionInterval, - []*proto.StatsEntity{}, + []*metrics.StatsEntityWrapper{}, } logs := binary.GetErrorLogs() @@ -101,7 +101,7 @@ func NewNginxErrorLog( return nginxErrorLog } -func (c *NginxErrorLog) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxErrorLog) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.collectLogStats(ctx, m) @@ -145,7 +145,7 @@ func (c *NginxErrorLog) Update(dimensions *metrics.CommonDim, collectorConf *met } } -func (c *NginxErrorLog) collectLogStats(ctx context.Context, m chan<- *proto.StatsEntity) { +func (c *NginxErrorLog) collectLogStats(ctx context.Context, m chan<- *metrics.StatsEntityWrapper) { c.mu.Lock() defer c.mu.Unlock() logs := c.binary.GetErrorLogs() @@ -174,7 +174,7 @@ func (c *NginxErrorLog) collectLogStats(ctx context.Context, m chan<- *proto.Sta for _, stat := range c.buf { m <- stat } - c.buf = []*proto.StatsEntity{} + c.buf = []*metrics.StatsEntityWrapper{} } func (c *NginxErrorLog) logStats(ctx context.Context, logFile string) { @@ -231,7 +231,7 @@ func (c *NginxErrorLog) logStats(ctx context.Context, logFile string) { UpstreamResponseFailedMetricName: 0, } - c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics)) + c.buf = append(c.buf, metrics.NewStatsEntityWrapper(c.baseDimensions.ToDimensions(), simpleMetrics, proto.MetricsReport_INSTANCE)) mu.Unlock() diff --git a/src/core/metrics/sources/nginx_error_log_test.go b/src/core/metrics/sources/nginx_error_log_test.go index 8e5e4bdb9..0141c30a8 100644 --- a/src/core/metrics/sources/nginx_error_log_test.go +++ b/src/core/metrics/sources/nginx_error_log_test.go @@ -133,11 +133,11 @@ func TestErrorLogStats(t *testing.T) { sort.SliceStable(test.expectedStats.GetSimplemetrics(), func(i, j int) bool { return test.expectedStats.GetSimplemetrics()[i].Name < test.expectedStats.GetSimplemetrics()[j].Name }) - sort.SliceStable(nginxErrorLog.buf[0].GetSimplemetrics(), func(i, j int) bool { - return nginxErrorLog.buf[0].GetSimplemetrics()[i].Name < nginxErrorLog.buf[0].GetSimplemetrics()[j].Name + sort.SliceStable(nginxErrorLog.buf[0].Data.GetSimplemetrics(), func(i, j int) bool { + return nginxErrorLog.buf[0].Data.GetSimplemetrics()[i].Name < nginxErrorLog.buf[0].Data.GetSimplemetrics()[j].Name }) - assert.Equal(tt, test.expectedStats.GetSimplemetrics(), nginxErrorLog.buf[0].GetSimplemetrics()) + assert.Equal(tt, test.expectedStats.GetSimplemetrics(), nginxErrorLog.buf[0].Data.GetSimplemetrics()) }) } } diff --git a/src/core/metrics/sources/nginx_oss.go b/src/core/metrics/sources/nginx_oss.go index 51d021165..776e5288b 100644 --- a/src/core/metrics/sources/nginx_oss.go +++ b/src/core/metrics/sources/nginx_oss.go @@ -34,7 +34,7 @@ func NewNginxOSS(baseDimensions *metrics.CommonDim, namespace, stubStatus string return &NginxOSS{baseDimensions: baseDimensions, stubStatus: stubStatus, namedMetric: &namedMetric{namespace: namespace}, logger: NewMetricSourceLogger()} } -func (c *NginxOSS) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxOSS) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.init.Do(func() { cl, err := client.NewNginxClient(&http.Client{}, c.stubStatus) @@ -108,7 +108,7 @@ func (c *NginxOSS) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *pr select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics): + case m <- metrics.NewStatsEntityWrapper(c.baseDimensions.ToDimensions(), simpleMetrics, proto.MetricsReport_INSTANCE): } c.prevStats = stats diff --git a/src/core/metrics/sources/nginx_oss_test.go b/src/core/metrics/sources/nginx_oss_test.go index 12db1bccd..919551443 100644 --- a/src/core/metrics/sources/nginx_oss_test.go +++ b/src/core/metrics/sources/nginx_oss_test.go @@ -66,14 +66,14 @@ Reading: 0 Writing: 1 Waiting: 0 name string namedMetric *namedMetric stubAPI string - m chan *proto.StatsEntity + m chan *metrics.StatsEntityWrapper expectedMetrics map[string]float64 }{ { "valid stub API", &namedMetric{namespace: "nginx", group: "http"}, server.URL + "/basic_status", - make(chan *proto.StatsEntity, 1), + make(chan *metrics.StatsEntityWrapper, 1), map[string]float64{ "nginx.status": float64(1), "nginx.http.conn.active": float64(1), @@ -91,7 +91,7 @@ Reading: 0 Writing: 1 Waiting: 0 "unknown stub API", &namedMetric{namespace: "nginx", group: "http"}, server.URL + "/unknown", - make(chan *proto.StatsEntity, 1), + make(chan *metrics.StatsEntityWrapper, 1), map[string]float64{ "nginx.status": float64(0), }, @@ -115,8 +115,8 @@ Reading: 0 Writing: 1 Waiting: 0 go c.Collect(ctx, wg, test.m) wg.Wait() statEntity := <-test.m - assert.Len(tt, statEntity.Simplemetrics, len(test.expectedMetrics)) - for _, metric := range statEntity.Simplemetrics { + assert.Len(tt, statEntity.Data.Simplemetrics, len(test.expectedMetrics)) + for _, metric := range statEntity.Data.Simplemetrics { assert.Contains(t, test.expectedMetrics, metric.Name) assert.Equal(t, test.expectedMetrics[metric.Name], metric.Value) } diff --git a/src/core/metrics/sources/nginx_plus.go b/src/core/metrics/sources/nginx_plus.go index 9a7b529ee..6b0e19b1c 100644 --- a/src/core/metrics/sources/nginx_plus.go +++ b/src/core/metrics/sources/nginx_plus.go @@ -50,7 +50,7 @@ func NewNginxPlus(baseDimensions *metrics.CommonDim, nginxNamespace, plusNamespa return &NginxPlus{baseDimensions: baseDimensions, nginxNamespace: nginxNamespace, plusNamespace: plusNamespace, plusAPI: plusAPI, clientVersion: clientVersion, logger: NewMetricSourceLogger()} } -func (c *NginxPlus) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxPlus) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.init.Do(func() { cl, err := plusclient.NewNginxClientWithVersion(&http.Client{}, c.plusAPI, c.clientVersion) @@ -106,7 +106,7 @@ func (c *NginxPlus) Stop() { log.Debugf("Stopping NginxPlus source for nginx id: %v", c.baseDimensions.NginxId) } -func (c *NginxPlus) sendMetrics(ctx context.Context, m chan<- *proto.StatsEntity, entries ...*proto.StatsEntity) { +func (c *NginxPlus) sendMetrics(ctx context.Context, m chan<- *metrics.StatsEntityWrapper, entries ...*metrics.StatsEntityWrapper) { for _, entry := range entries { select { case <-ctx.Done(): @@ -116,7 +116,7 @@ func (c *NginxPlus) sendMetrics(ctx context.Context, m chan<- *proto.StatsEntity } } -func (c *NginxPlus) collectMetrics(stats, prevStats *plusclient.Stats) (entries []*proto.StatsEntity) { +func (c *NginxPlus) collectMetrics(stats, prevStats *plusclient.Stats) (entries []*metrics.StatsEntityWrapper) { entries = append(entries, c.instanceMetrics(stats, prevStats), c.commonMetrics(stats, prevStats), @@ -124,16 +124,17 @@ func (c *NginxPlus) collectMetrics(stats, prevStats *plusclient.Stats) (entries entries = append(entries, c.serverZoneMetrics(stats, prevStats)...) entries = append(entries, c.streamServerZoneMetrics(stats, prevStats)...) entries = append(entries, c.locationZoneMetrics(stats, prevStats)...) - entries = append(entries, c.cacheMetrics(stats, prevStats)...) - entries = append(entries, c.httpUpstreamMetrics(stats, prevStats)...) - entries = append(entries, c.streamUpstreamMetrics(stats, prevStats)...) entries = append(entries, c.slabMetrics(stats, prevStats)...) entries = append(entries, c.httpLimitConnsMetrics(stats, prevStats)...) entries = append(entries, c.httpLimitRequestMetrics(stats, prevStats)...) - return entries + entries = append(entries, c.cacheMetrics(stats, prevStats)...) + entries = append(entries, c.httpUpstreamMetrics(stats, prevStats)...) + entries = append(entries, c.streamUpstreamMetrics(stats, prevStats)...) + + return } -func (c *NginxPlus) instanceMetrics(stats, prevStats *plusclient.Stats) *proto.StatsEntity { +func (c *NginxPlus) instanceMetrics(stats, prevStats *plusclient.Stats) *metrics.StatsEntityWrapper { l := &namedMetric{namespace: c.nginxNamespace, group: ""} configGeneration := stats.NginxInfo.Generation - prevStats.NginxInfo.Generation @@ -147,11 +148,11 @@ func (c *NginxPlus) instanceMetrics(stats, prevStats *plusclient.Stats) *proto.S }) dims := c.baseDimensions.ToDimensions() - return metrics.NewStatsEntity(dims, simpleMetrics) + return metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE) } // commonMetrics uses the namespace of nginx because the metrics are common between oss and plus -func (c *NginxPlus) commonMetrics(stats, prevStats *plusclient.Stats) *proto.StatsEntity { +func (c *NginxPlus) commonMetrics(stats, prevStats *plusclient.Stats) *metrics.StatsEntityWrapper { l := &namedMetric{namespace: c.nginxNamespace, group: "http"} // For the case if nginx restarted (systemctl restart nginx), the stats counters were reset to zero, and @@ -182,10 +183,10 @@ func (c *NginxPlus) commonMetrics(stats, prevStats *plusclient.Stats) *proto.Sta }) dims := c.baseDimensions.ToDimensions() - return metrics.NewStatsEntity(dims, simpleMetrics) + return metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE) } -func (c *NginxPlus) sslMetrics(stats, prevStats *plusclient.Stats) *proto.StatsEntity { +func (c *NginxPlus) sslMetrics(stats, prevStats *plusclient.Stats) *metrics.StatsEntityWrapper { l := &namedMetric{namespace: c.plusNamespace, group: ""} sslHandshakes := stats.SSL.Handshakes - prevStats.SSL.Handshakes @@ -208,11 +209,11 @@ func (c *NginxPlus) sslMetrics(stats, prevStats *plusclient.Stats) *proto.StatsE }) dims := c.baseDimensions.ToDimensions() - return metrics.NewStatsEntity(dims, simpleMetrics) + return metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE) } -func (c *NginxPlus) serverZoneMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - zoneMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) serverZoneMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + zoneMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, sz := range stats.ServerZones { l := &namedMetric{namespace: c.plusNamespace, group: "http"} @@ -273,14 +274,14 @@ func (c *NginxPlus) serverZoneMetrics(stats, prevStats *plusclient.Stats) []*pro dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "server_zone", Value: name}) - zoneMetrics = append(zoneMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + zoneMetrics = append(zoneMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE)) } return zoneMetrics } -func (c *NginxPlus) streamServerZoneMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - zoneMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) streamServerZoneMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + zoneMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, ssz := range stats.StreamServerZones { l := &namedMetric{namespace: c.plusNamespace, group: "stream"} @@ -331,13 +332,13 @@ func (c *NginxPlus) streamServerZoneMetrics(stats, prevStats *plusclient.Stats) dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "server_zone", Value: name}) - zoneMetrics = append(zoneMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + zoneMetrics = append(zoneMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE)) } return zoneMetrics } -func (c *NginxPlus) locationZoneMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - zoneMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) locationZoneMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + zoneMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, lz := range stats.LocationZones { l := &namedMetric{namespace: c.plusNamespace, group: "http"} @@ -397,14 +398,14 @@ func (c *NginxPlus) locationZoneMetrics(stats, prevStats *plusclient.Stats) []*p dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "location_zone", Value: name}) - zoneMetrics = append(zoneMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + zoneMetrics = append(zoneMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE)) } return zoneMetrics } -func (c *NginxPlus) httpUpstreamMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - upstreamMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) httpUpstreamMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + upstreamMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, u := range stats.Upstreams { httpUpstreamHeaderTimes := []float64{} httpUpstreamResponseTimes := []float64{} @@ -496,7 +497,7 @@ func (c *NginxPlus) httpUpstreamMetrics(stats, prevStats *plusclient.Stats) []*p peerDims = append(peerDims, &proto.Dimension{Name: "upstream_zone", Value: u.Zone}) peerDims = append(peerDims, &proto.Dimension{Name: "peer.name", Value: peer.Name}) peerDims = append(peerDims, &proto.Dimension{Name: "peer.address", Value: peer.Server}) - upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntity(peerDims, simpleMetrics2)) + upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntityWrapper(peerDims, simpleMetrics2, proto.MetricsReport_UPSTREAMS)) } upstreamQueueOverflows := u.Queue.Overflows - prevStats.Upstreams[name].Queue.Overflows @@ -529,14 +530,14 @@ func (c *NginxPlus) httpUpstreamMetrics(stats, prevStats *plusclient.Stats) []*p upstreamDims := c.baseDimensions.ToDimensions() upstreamDims = append(upstreamDims, &proto.Dimension{Name: "upstream", Value: name}) upstreamDims = append(upstreamDims, &proto.Dimension{Name: "upstream_zone", Value: u.Zone}) - upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntity(upstreamDims, simpleMetrics)) + upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntityWrapper(upstreamDims, simpleMetrics, proto.MetricsReport_UPSTREAMS)) } return upstreamMetrics } -func (c *NginxPlus) streamUpstreamMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - upstreamMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) streamUpstreamMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + upstreamMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, u := range stats.StreamUpstreams { streamUpstreamResponseTimes := []float64{} streamUpstreamConnTimes := []float64{} @@ -605,7 +606,7 @@ func (c *NginxPlus) streamUpstreamMetrics(stats, prevStats *plusclient.Stats) [] peerDims = append(peerDims, &proto.Dimension{Name: "upstream_zone", Value: u.Zone}) peerDims = append(peerDims, &proto.Dimension{Name: "peer.name", Value: peer.Name}) peerDims = append(peerDims, &proto.Dimension{Name: "peer.address", Value: peer.Server}) - upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntity(peerDims, simpleMetrics2)) + upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntityWrapper(peerDims, simpleMetrics2, proto.MetricsReport_UPSTREAMS)) } simpleMetrics := l.convertSamplesToSimpleMetrics(map[string]float64{ @@ -629,14 +630,14 @@ func (c *NginxPlus) streamUpstreamMetrics(stats, prevStats *plusclient.Stats) [] upstreamDims := c.baseDimensions.ToDimensions() upstreamDims = append(upstreamDims, &proto.Dimension{Name: "upstream", Value: name}) upstreamDims = append(upstreamDims, &proto.Dimension{Name: "upstream_zone", Value: u.Zone}) - upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntity(upstreamDims, simpleMetrics)) + upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntityWrapper(upstreamDims, simpleMetrics, proto.MetricsReport_UPSTREAMS)) } return upstreamMetrics } -func (c *NginxPlus) cacheMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - zoneMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) cacheMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + zoneMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, ca := range stats.Caches { l := &namedMetric{namespace: c.plusNamespace, group: "cache"} @@ -718,15 +719,15 @@ func (c *NginxPlus) cacheMetrics(stats, prevStats *plusclient.Stats) []*proto.St dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "cache_zone", Value: name}) - zoneMetrics = append(zoneMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + zoneMetrics = append(zoneMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_CACHE_ZONE)) } return zoneMetrics } -func (c *NginxPlus) slabMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { +func (c *NginxPlus) slabMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { l := &namedMetric{namespace: c.plusNamespace, group: ""} - slabMetrics := make([]*proto.StatsEntity, 0) + slabMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, slab := range stats.Slabs { pages := slab.Pages @@ -746,7 +747,7 @@ func (c *NginxPlus) slabMetrics(stats, prevStats *plusclient.Stats) []*proto.Sta dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "zone", Value: name}) - slabMetrics = append(slabMetrics, metrics.NewStatsEntity(dims, slabSimpleMetrics)) + slabMetrics = append(slabMetrics, metrics.NewStatsEntityWrapper(dims, slabSimpleMetrics, proto.MetricsReport_INSTANCE)) for slotNum, slot := range slab.Slots { slotSimpleMetrics := l.convertSamplesToSimpleMetrics(map[string]float64{ @@ -755,15 +756,15 @@ func (c *NginxPlus) slabMetrics(stats, prevStats *plusclient.Stats) []*proto.Sta "slab.slots." + slotNum + ".reqs": float64(slot.Reqs), "slab.slots." + slotNum + ".used": float64(slot.Used), }) - slabMetrics = append(slabMetrics, metrics.NewStatsEntity(dims, slotSimpleMetrics)) + slabMetrics = append(slabMetrics, metrics.NewStatsEntityWrapper(dims, slotSimpleMetrics, proto.MetricsReport_INSTANCE)) } } return slabMetrics } -func (c *NginxPlus) httpLimitConnsMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - limitConnsMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) httpLimitConnsMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + limitConnsMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, lc := range stats.HTTPLimitConnections { l := &namedMetric{namespace: c.plusNamespace, group: "http"} @@ -774,14 +775,14 @@ func (c *NginxPlus) httpLimitConnsMetrics(stats, prevStats *plusclient.Stats) [] }) dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "limit_conn_zone", Value: name}) - limitConnsMetrics = append(limitConnsMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + limitConnsMetrics = append(limitConnsMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE)) } return limitConnsMetrics } -func (c *NginxPlus) httpLimitRequestMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - limitRequestMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) httpLimitRequestMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + limitRequestMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, lr := range stats.HTTPLimitRequests { l := &namedMetric{namespace: c.plusNamespace, group: "http"} @@ -794,7 +795,7 @@ func (c *NginxPlus) httpLimitRequestMetrics(stats, prevStats *plusclient.Stats) }) dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "limit_req_zone", Value: name}) - limitRequestMetrics = append(limitRequestMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + limitRequestMetrics = append(limitRequestMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE)) } return limitRequestMetrics diff --git a/src/core/metrics/sources/nginx_plus_test.go b/src/core/metrics/sources/nginx_plus_test.go index b7ca117ba..701f5e871 100644 --- a/src/core/metrics/sources/nginx_plus_test.go +++ b/src/core/metrics/sources/nginx_plus_test.go @@ -79,7 +79,7 @@ type FakeNginxPlus struct { } // Collect is fake collector that hard codes a stats struct response to avoid dependency on external NGINX Plus api -func (f *FakeNginxPlus) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (f *FakeNginxPlus) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() stats := plusclient.Stats{ HTTPRequests: plusclient.HTTPRequests{ @@ -706,11 +706,11 @@ func TestNginxPlus_Collect(t *testing.T) { } tests := []struct { baseDimensions *metrics.CommonDim - m chan *proto.StatsEntity + m chan *metrics.StatsEntityWrapper }{ { baseDimensions: metrics.NewCommonDim(hostInfo, &config.Config{}, ""), - m: make(chan *proto.StatsEntity, 127), + m: make(chan *metrics.StatsEntityWrapper, 127), }, } @@ -724,8 +724,8 @@ func TestNginxPlus_Collect(t *testing.T) { wg.Wait() instanceMetrics := <-test.m - assert.Len(t, instanceMetrics.Simplemetrics, len(expectedInstanceMetrics)) - for _, metric := range instanceMetrics.Simplemetrics { + assert.Len(t, instanceMetrics.Data.Simplemetrics, len(expectedInstanceMetrics)) + for _, metric := range instanceMetrics.Data.Simplemetrics { assert.Contains(t, expectedInstanceMetrics, metric.Name) switch metric.Name { case "nginx.status": @@ -736,8 +736,8 @@ func TestNginxPlus_Collect(t *testing.T) { } commonMetrics := <-test.m - assert.Len(t, commonMetrics.Simplemetrics, len(expectedCommonMetrics)) - for _, metric := range commonMetrics.Simplemetrics { + assert.Len(t, commonMetrics.Data.Simplemetrics, len(expectedCommonMetrics)) + for _, metric := range commonMetrics.Data.Simplemetrics { assert.Contains(t, expectedCommonMetrics, metric.Name) switch metric.Name { case "nginx.http.request.count": // delta of sum @@ -748,20 +748,20 @@ func TestNginxPlus_Collect(t *testing.T) { } sslMetrics := <-test.m - assert.Len(t, sslMetrics.Simplemetrics, len(expectedSSLMetrics)) - for _, metric := range sslMetrics.Simplemetrics { + assert.Len(t, sslMetrics.Data.Simplemetrics, len(expectedSSLMetrics)) + for _, metric := range sslMetrics.Data.Simplemetrics { assert.Contains(t, expectedSSLMetrics, metric.Name) } serverZoneMetrics := <-test.m - assert.Len(t, serverZoneMetrics.Simplemetrics, len(expectedServerZoneMetrics)) - for _, dimension := range serverZoneMetrics.Dimensions { + assert.Len(t, serverZoneMetrics.Data.Simplemetrics, len(expectedServerZoneMetrics)) + for _, dimension := range serverZoneMetrics.Data.Dimensions { switch dimension.Name { case "server_zone": assert.Equal(t, serverZoneName, dimension.Value) } } - for _, metric := range serverZoneMetrics.Simplemetrics { + for _, metric := range serverZoneMetrics.Data.Simplemetrics { assert.Contains(t, expectedServerZoneMetrics, metric.Name) switch metric.Name { case "plus.http.request.count": @@ -778,14 +778,14 @@ func TestNginxPlus_Collect(t *testing.T) { } streamServerZoneMetrics := <-test.m - assert.Len(t, streamServerZoneMetrics.Simplemetrics, len(expectedStreamServerZoneMetrics)) - for _, dimension := range streamServerZoneMetrics.Dimensions { + assert.Len(t, streamServerZoneMetrics.Data.Simplemetrics, len(expectedStreamServerZoneMetrics)) + for _, dimension := range streamServerZoneMetrics.Data.Dimensions { switch dimension.Name { case "server_zone": assert.Equal(t, streamServerZoneName, dimension.Value) } } - for _, metric := range streamServerZoneMetrics.Simplemetrics { + for _, metric := range streamServerZoneMetrics.Data.Simplemetrics { assert.Contains(t, expectedStreamServerZoneMetrics, metric.Name) switch metric.Name { case "plus.stream.connections": @@ -802,14 +802,14 @@ func TestNginxPlus_Collect(t *testing.T) { } locationZoneMetrics := <-test.m - assert.Len(t, locationZoneMetrics.Simplemetrics, len(expectedLocationZoneMetrics)) - for _, dimension := range locationZoneMetrics.Dimensions { + assert.Len(t, locationZoneMetrics.Data.Simplemetrics, len(expectedLocationZoneMetrics)) + for _, dimension := range locationZoneMetrics.Data.Dimensions { switch dimension.Name { case "location_zone": assert.Equal(t, locationZoneName, dimension.Value) } } - for _, metric := range locationZoneMetrics.Simplemetrics { + for _, metric := range locationZoneMetrics.Data.Simplemetrics { assert.Contains(t, expectedLocationZoneMetrics, metric.Name) switch metric.Name { case "plus.http.request.count": @@ -825,15 +825,89 @@ func TestNginxPlus_Collect(t *testing.T) { } } + slabMetrics := <-test.m + assert.Len(t, slabMetrics.Data.Simplemetrics, len(expectedSlabMetrics)) + for _, dimension := range slabMetrics.Data.Dimensions { + switch dimension.Name { + case "zone": + assert.Equal(t, serverZoneName, dimension.Value) + } + } + for _, metric := range slabMetrics.Data.Simplemetrics { + assert.Contains(t, expectedSlabMetrics, metric.Name) + switch metric.Name { + case "plus.slab.pages.used": + assert.Equal(t, expectedSlabMetrics["plus.slab.pages.used"], metric.Value) + case "plus.slab.pages.free": + assert.Equal(t, expectedSlabMetrics["plus.slab.pages.free"], metric.Value) + case "plus.slab.pages.total": + assert.Equal(t, expectedSlabMetrics["plus.slab.pages.total"], metric.Value) + case "plus.slab.pages.pct_used": + assert.Equal(t, expectedSlabMetrics["plus.slab.pages.pct_used"], metric.Value) + } + } + + slabSlotsMetrics := <-test.m + assert.Len(t, slabSlotsMetrics.Data.Simplemetrics, len(expectedSlabSlotMetrics)) + for _, metric := range slabSlotsMetrics.Data.Simplemetrics { + assert.Contains(t, expectedSlabSlotMetrics, metric.Name) + assert.Equal(t, expectedSlabSlotMetrics[metric.Name], metric.Value) + } + + limitConnectionsMetrics := <-test.m + assert.Len(t, limitConnectionsMetrics.Data.Simplemetrics, len(expectedHTTPLimitConnsMetrics)) + for _, dimension := range limitConnectionsMetrics.Data.Dimensions { + switch dimension.Name { + case "limit_conn_zone": + assert.Equal(t, limitConnectionsName, dimension.Value) + } + } + for _, metric := range limitConnectionsMetrics.Data.Simplemetrics { + assert.Contains(t, expectedHTTPLimitConnsMetrics, metric.Name) + switch metric.Name { + case "plus.http.limit_conns.passed": + assert.Equal(t, expectedHTTPLimitConnsMetrics["plus.http.limit_conns.passed"], metric.Value) + case "plus.http.limit_conns.rejected": + assert.Equal(t, expectedHTTPLimitConnsMetrics["plus.http.limit_conns.rejected"], metric.Value) + case "plus.http.limit_conns.rejected_dry_run": + assert.Equal(t, expectedHTTPLimitConnsMetrics["plus.http.limit_conns.rejected_dry_run"], metric.Value) + } + } + + limitRequestsMetrics := <-test.m + assert.Len(t, limitRequestsMetrics.Data.Simplemetrics, len(expectedHTTPLimitReqsMetrics)) + for _, dimension := range limitRequestsMetrics.Data.Dimensions { + switch dimension.Name { + case "limit_req_zone": + assert.Equal(t, limitRequestName, dimension.Value) + + } + } + for _, metric := range limitRequestsMetrics.Data.Simplemetrics { + assert.Contains(t, expectedHTTPLimitReqsMetrics, metric.Name) + switch metric.Name { + case "plus.http.limit_reqs.passed": + assert.Equal(t, expectedHTTPLimitReqsMetrics["plus.http.limit_reqs.passed"], metric.Value) + case "plus.http.limit_reqs.delayed": + assert.Equal(t, expectedHTTPLimitReqsMetrics["plus.http.limit_reqs.delayed"], metric.Value) + case "plus.http.limit_reqs.rejected": + assert.Equal(t, expectedHTTPLimitReqsMetrics["plus.http.limit_reqs.rejected"], metric.Value) + case "plus.http.limit_reqs.delayed_dry_run": + assert.Equal(t, expectedHTTPLimitReqsMetrics["plus.http.limit_reqs.delayed_dry_run"], metric.Value) + case "plus.http.limit_reqs.rejected_dry_run": + assert.Equal(t, expectedHTTPLimitReqsMetrics["plus.http.limit_reqs.rejected_dry_run"], metric.Value) + } + } + cacheZoneMetrics := <-test.m - assert.Len(t, cacheZoneMetrics.Simplemetrics, len(expectedCacheZoneMetrics)) - for _, dimension := range cacheZoneMetrics.Dimensions { + assert.Len(t, cacheZoneMetrics.Data.Simplemetrics, len(expectedCacheZoneMetrics)) + for _, dimension := range cacheZoneMetrics.Data.Dimensions { switch dimension.Name { case "cache_zone": assert.Equal(t, cacheZoneName, dimension.Value) } } - for _, metric := range cacheZoneMetrics.Simplemetrics { + for _, metric := range cacheZoneMetrics.Data.Simplemetrics { assert.Contains(t, expectedCacheZoneMetrics, metric.Name) switch metric.Name { case "plus.cache.size": @@ -852,8 +926,8 @@ func TestNginxPlus_Collect(t *testing.T) { } httpPeer1upstreamMetrics := <-test.m - assert.Len(t, httpPeer1upstreamMetrics.Simplemetrics, len(expectedHttpPeer1UpstreamMetrics)) - for _, dimension := range httpPeer1upstreamMetrics.Dimensions { + assert.Len(t, httpPeer1upstreamMetrics.Data.Simplemetrics, len(expectedHttpPeer1UpstreamMetrics)) + for _, dimension := range httpPeer1upstreamMetrics.Data.Dimensions { switch dimension.Name { case "upstream": assert.Equal(t, upstreamName, dimension.Value) @@ -865,7 +939,7 @@ func TestNginxPlus_Collect(t *testing.T) { assert.Equal(t, upstreamPeer1ServerAddress, dimension.Value) } } - for _, metric := range httpPeer1upstreamMetrics.Simplemetrics { + for _, metric := range httpPeer1upstreamMetrics.Data.Simplemetrics { assert.Contains(t, expectedHttpPeer1UpstreamMetrics, metric.Name) switch metric.Name { case "plus.http.upstream.peers.header_time": @@ -888,8 +962,8 @@ func TestNginxPlus_Collect(t *testing.T) { } httpPeer2upstreamMetrics := <-test.m - assert.Len(t, httpPeer2upstreamMetrics.Simplemetrics, len(expectedHttpPeer2UpstreamMetrics)) - for _, dimension := range httpPeer2upstreamMetrics.Dimensions { + assert.Len(t, httpPeer2upstreamMetrics.Data.Simplemetrics, len(expectedHttpPeer2UpstreamMetrics)) + for _, dimension := range httpPeer2upstreamMetrics.Data.Dimensions { switch dimension.Name { case "upstream": assert.Equal(t, upstreamName, dimension.Value) @@ -901,7 +975,7 @@ func TestNginxPlus_Collect(t *testing.T) { assert.Equal(t, upstreamPeer2ServerAddress, dimension.Value) } } - for _, metric := range httpPeer2upstreamMetrics.Simplemetrics { + for _, metric := range httpPeer2upstreamMetrics.Data.Simplemetrics { assert.Contains(t, expectedHttpPeer2UpstreamMetrics, metric.Name) switch metric.Name { case "plus.http.upstream.peers.header_time": @@ -924,8 +998,8 @@ func TestNginxPlus_Collect(t *testing.T) { } httpUpstreamMetrics := <-test.m - assert.Len(t, httpUpstreamMetrics.Simplemetrics, len(expectedHttpUpstreamMetrics)) - for _, dimension := range httpUpstreamMetrics.Dimensions { + assert.Len(t, httpUpstreamMetrics.Data.Simplemetrics, len(expectedHttpUpstreamMetrics)) + for _, dimension := range httpUpstreamMetrics.Data.Dimensions { switch dimension.Name { case "upstream": assert.Equal(t, upstreamName, dimension.Value) @@ -933,7 +1007,7 @@ func TestNginxPlus_Collect(t *testing.T) { assert.Equal(t, upstreamZoneName, dimension.Value) } } - for _, metric := range httpUpstreamMetrics.Simplemetrics { + for _, metric := range httpUpstreamMetrics.Data.Simplemetrics { assert.Contains(t, expectedHttpUpstreamMetrics, metric.Name) switch metric.Name { case "plus.http.upstream.queue.maxsize": @@ -964,8 +1038,8 @@ func TestNginxPlus_Collect(t *testing.T) { } streamPeer1upstreamMetrics := <-test.m - assert.Len(t, streamPeer1upstreamMetrics.Simplemetrics, len(expectedStreamPeer1UpstreamMetrics)) - for _, dimension := range streamPeer1upstreamMetrics.Dimensions { + assert.Len(t, streamPeer1upstreamMetrics.Data.Simplemetrics, len(expectedStreamPeer1UpstreamMetrics)) + for _, dimension := range streamPeer1upstreamMetrics.Data.Dimensions { switch dimension.Name { case "upstream": assert.Equal(t, upstreamName, dimension.Value) @@ -979,8 +1053,8 @@ func TestNginxPlus_Collect(t *testing.T) { } streamPeer2upstreamMetrics := <-test.m - assert.Len(t, streamPeer2upstreamMetrics.Simplemetrics, len(expectedStreamPeer2UpstreamMetrics)) - for _, dimension := range streamPeer2upstreamMetrics.Dimensions { + assert.Len(t, streamPeer2upstreamMetrics.Data.Simplemetrics, len(expectedStreamPeer2UpstreamMetrics)) + for _, dimension := range streamPeer2upstreamMetrics.Data.Dimensions { switch dimension.Name { case "upstream": assert.Equal(t, upstreamName, dimension.Value) @@ -993,7 +1067,7 @@ func TestNginxPlus_Collect(t *testing.T) { } } - for _, metric := range streamPeer1upstreamMetrics.Simplemetrics { + for _, metric := range streamPeer1upstreamMetrics.Data.Simplemetrics { assert.Contains(t, expectedStreamPeer1UpstreamMetrics, metric.Name) switch metric.Name { case "plus.stream.upstream.peers.conn.active": @@ -1015,7 +1089,7 @@ func TestNginxPlus_Collect(t *testing.T) { } } - for _, metric := range streamPeer2upstreamMetrics.Simplemetrics { + for _, metric := range streamPeer2upstreamMetrics.Data.Simplemetrics { assert.Contains(t, expectedStreamPeer2UpstreamMetrics, metric.Name) switch metric.Name { case "plus.stream.upstream.peers.conn.active": @@ -1038,8 +1112,8 @@ func TestNginxPlus_Collect(t *testing.T) { } streamUpstreamMetrics := <-test.m - assert.Len(t, streamUpstreamMetrics.Simplemetrics, len(expectedStreamUpstreamMetrics)) - for _, dimension := range streamUpstreamMetrics.Dimensions { + assert.Len(t, streamUpstreamMetrics.Data.Simplemetrics, len(expectedStreamUpstreamMetrics)) + for _, dimension := range streamUpstreamMetrics.Data.Dimensions { switch dimension.Name { case "upstream": assert.Equal(t, upstreamName, dimension.Value) @@ -1047,7 +1121,7 @@ func TestNginxPlus_Collect(t *testing.T) { assert.Equal(t, upstreamZoneName, dimension.Value) } } - for _, metric := range streamUpstreamMetrics.Simplemetrics { + for _, metric := range streamUpstreamMetrics.Data.Simplemetrics { assert.Contains(t, expectedStreamUpstreamMetrics, metric.Name) switch metric.Name { case "plus.stream.upstream.zombies": @@ -1075,81 +1149,7 @@ func TestNginxPlus_Collect(t *testing.T) { } } - slabMetrics := <-test.m - assert.Len(t, slabMetrics.Simplemetrics, len(expectedSlabMetrics)) - for _, dimension := range slabMetrics.Dimensions { - switch dimension.Name { - case "zone": - assert.Equal(t, serverZoneName, dimension.Value) - } - } - for _, metric := range slabMetrics.Simplemetrics { - assert.Contains(t, expectedSlabMetrics, metric.Name) - switch metric.Name { - case "plus.slab.pages.used": - assert.Equal(t, expectedSlabMetrics["plus.slab.pages.used"], metric.Value) - case "plus.slab.pages.free": - assert.Equal(t, expectedSlabMetrics["plus.slab.pages.free"], metric.Value) - case "plus.slab.pages.total": - assert.Equal(t, expectedSlabMetrics["plus.slab.pages.total"], metric.Value) - case "plus.slab.pages.pct_used": - assert.Equal(t, expectedSlabMetrics["plus.slab.pages.pct_used"], metric.Value) - } - } - - slabSlotsMetrics := <-test.m - assert.Len(t, slabSlotsMetrics.Simplemetrics, len(expectedSlabSlotMetrics)) - for _, metric := range slabSlotsMetrics.Simplemetrics { - assert.Contains(t, expectedSlabSlotMetrics, metric.Name) - assert.Equal(t, expectedSlabSlotMetrics[metric.Name], metric.Value) - } - - limitConnectionsMetrics := <-test.m - assert.Len(t, limitConnectionsMetrics.Simplemetrics, len(expectedHTTPLimitConnsMetrics)) - for _, dimension := range limitConnectionsMetrics.Dimensions { - switch dimension.Name { - case "limit_conn_zone": - assert.Equal(t, limitConnectionsName, dimension.Value) - } - } - for _, metric := range limitConnectionsMetrics.Simplemetrics { - assert.Contains(t, expectedHTTPLimitConnsMetrics, metric.Name) - switch metric.Name { - case "plus.http.limit_conns.passed": - assert.Equal(t, expectedHTTPLimitConnsMetrics["plus.http.limit_conns.passed"], metric.Value) - case "plus.http.limit_conns.rejected": - assert.Equal(t, expectedHTTPLimitConnsMetrics["plus.http.limit_conns.rejected"], metric.Value) - case "plus.http.limit_conns.rejected_dry_run": - assert.Equal(t, expectedHTTPLimitConnsMetrics["plus.http.limit_conns.rejected_dry_run"], metric.Value) - } - } - - limitRequestsMetrics := <-test.m - assert.Len(t, limitRequestsMetrics.Simplemetrics, len(expectedHTTPLimitReqsMetrics)) - for _, dimension := range limitRequestsMetrics.Dimensions { - switch dimension.Name { - case "limit_req_zone": - assert.Equal(t, limitRequestName, dimension.Value) - - } - } - for _, metric := range limitRequestsMetrics.Simplemetrics { - assert.Contains(t, expectedHTTPLimitReqsMetrics, metric.Name) - switch metric.Name { - case "plus.http.limit_reqs.passed": - assert.Equal(t, expectedHTTPLimitReqsMetrics["plus.http.limit_reqs.passed"], metric.Value) - case "plus.http.limit_reqs.delayed": - assert.Equal(t, expectedHTTPLimitReqsMetrics["plus.http.limit_reqs.delayed"], metric.Value) - case "plus.http.limit_reqs.rejected": - assert.Equal(t, expectedHTTPLimitReqsMetrics["plus.http.limit_reqs.rejected"], metric.Value) - case "plus.http.limit_reqs.delayed_dry_run": - assert.Equal(t, expectedHTTPLimitReqsMetrics["plus.http.limit_reqs.delayed_dry_run"], metric.Value) - case "plus.http.limit_reqs.rejected_dry_run": - assert.Equal(t, expectedHTTPLimitReqsMetrics["plus.http.limit_reqs.rejected_dry_run"], metric.Value) - } - } - - var extraMetrics []*proto.StatsEntity + var extraMetrics []*metrics.StatsEntityWrapper EMWAIT: for { select { diff --git a/src/core/metrics/sources/nginx_process.go b/src/core/metrics/sources/nginx_process.go index 35c55810f..4abc9f6c8 100644 --- a/src/core/metrics/sources/nginx_process.go +++ b/src/core/metrics/sources/nginx_process.go @@ -43,7 +43,7 @@ func (c *NginxProcess) getNginxCount() float64 { return 0.0 } -func (c *NginxProcess) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxProcess) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() l := &namedMetric{namespace: PlusNamespace, group: ""} @@ -53,7 +53,7 @@ func (c *NginxProcess) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), countSimpleMetric): + case m <- metrics.NewStatsEntityWrapper(c.baseDimensions.ToDimensions(), countSimpleMetric, proto.MetricsReport_INSTANCE): } } diff --git a/src/core/metrics/sources/nginx_process_test.go b/src/core/metrics/sources/nginx_process_test.go index 9d5001415..f8c8b4543 100644 --- a/src/core/metrics/sources/nginx_process_test.go +++ b/src/core/metrics/sources/nginx_process_test.go @@ -60,7 +60,7 @@ func TestNginxProcessCollector_Collect_Process(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) - m := make(chan *proto.StatsEntity) + m := make(chan *metrics.StatsEntityWrapper) go n.Collect(ctx, &wg, m) time.Sleep(100 * time.Millisecond) @@ -70,7 +70,7 @@ func TestNginxProcessCollector_Collect_Process(t *testing.T) { // the prev stats as equal to the initial stats collected // that's ok, but we should test the counter gauge computations again later metricReport := <-m - for _, metric := range metricReport.Simplemetrics { + for _, metric := range metricReport.Data.Simplemetrics { switch metric.Name { case "plus.instance.count": assert.Equal(t, float64(1), metric.Value) @@ -104,7 +104,7 @@ func TestNginxProcessCollector_Collect_NoProcess(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) - m := make(chan *proto.StatsEntity) + m := make(chan *metrics.StatsEntityWrapper) go n.Collect(ctx, &wg, m) time.Sleep(100 * time.Millisecond) @@ -114,7 +114,7 @@ func TestNginxProcessCollector_Collect_NoProcess(t *testing.T) { // the prev stats as equal to the initial stats collected // that's ok, but we should test the counter gauge computations again later metricReport := <-m - for _, metric := range metricReport.Simplemetrics { + for _, metric := range metricReport.Data.Simplemetrics { switch metric.Name { case "plus.instance.count": assert.Equal(t, float64(0), metric.Value) @@ -151,7 +151,7 @@ func TestNginxProcessCollector_Collect_NotPlus(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) - m := make(chan *proto.StatsEntity) + m := make(chan *metrics.StatsEntityWrapper) go n.Collect(ctx, &wg, m) time.Sleep(100 * time.Millisecond) @@ -161,7 +161,7 @@ func TestNginxProcessCollector_Collect_NotPlus(t *testing.T) { // the prev stats as equal to the initial stats collected // that's ok, but we should test the counter gauge computations again later metricReport := <-m - for _, metric := range metricReport.Simplemetrics { + for _, metric := range metricReport.Data.Simplemetrics { switch metric.Name { case "plus.instance.count": assert.Equal(t, float64(0), metric.Value) diff --git a/src/core/metrics/sources/nginx_static.go b/src/core/metrics/sources/nginx_static.go index 86b0465ef..82cdde4f4 100644 --- a/src/core/metrics/sources/nginx_static.go +++ b/src/core/metrics/sources/nginx_static.go @@ -11,7 +11,6 @@ import ( "context" "sync" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core/metrics" log "github.com/sirupsen/logrus" ) @@ -27,7 +26,7 @@ func NewNginxStatic(baseDimensions *metrics.CommonDim, namespace string) *NginxS return &NginxStatic{baseDimensions: baseDimensions, namedMetric: &namedMetric{namespace: namespace}} } -func (c *NginxStatic) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxStatic) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() SendNginxDownStatus(ctx, c.baseDimensions.ToDimensions(), m) diff --git a/src/core/metrics/sources/nginx_static_test.go b/src/core/metrics/sources/nginx_static_test.go index 69254984a..02e3b7048 100644 --- a/src/core/metrics/sources/nginx_static_test.go +++ b/src/core/metrics/sources/nginx_static_test.go @@ -46,12 +46,12 @@ func TestNginxStatic_Collect(t *testing.T) { tests := []struct { name string namedMetric *namedMetric - m chan *proto.StatsEntity + m chan *metrics.StatsEntityWrapper }{ { "nginx static test", &namedMetric{namespace: "nginx"}, - make(chan *proto.StatsEntity, 1), + make(chan *metrics.StatsEntityWrapper, 1), }, } @@ -70,8 +70,8 @@ func TestNginxStatic_Collect(t *testing.T) { go c.Collect(ctx, wg, test.m) wg.Wait() statEntity := <-test.m - assert.Len(tt, statEntity.Simplemetrics, len(expectedMetrics)) - for _, metric := range statEntity.Simplemetrics { + assert.Len(tt, statEntity.Data.Simplemetrics, len(expectedMetrics)) + for _, metric := range statEntity.Data.Simplemetrics { assert.Contains(t, expectedMetrics, metric.Name) assert.Equal(t, expectedMetricsValues[metric.Name], metric.Value) } diff --git a/src/core/metrics/sources/nginx_worker.go b/src/core/metrics/sources/nginx_worker.go index a64be2d23..ff619e2c4 100644 --- a/src/core/metrics/sources/nginx_worker.go +++ b/src/core/metrics/sources/nginx_worker.go @@ -47,7 +47,7 @@ func NewNginxWorker(baseDimensions *metrics.CommonDim, } } -func (c *NginxWorker) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxWorker) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { var err error defer wg.Done() childProcs := c.binary.GetChildProcesses() @@ -122,7 +122,7 @@ func (c *NginxWorker) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics): + case m <- metrics.NewStatsEntityWrapper(c.baseDimensions.ToDimensions(), simpleMetrics, proto.MetricsReport_INSTANCE): } c.prevStats[pid] = stats diff --git a/src/core/metrics/sources/nginx_worker_test.go b/src/core/metrics/sources/nginx_worker_test.go index de5a1900a..1f9959a87 100644 --- a/src/core/metrics/sources/nginx_worker_test.go +++ b/src/core/metrics/sources/nginx_worker_test.go @@ -86,7 +86,7 @@ func TestNginxWorkerCollector(t *testing.T) { wg := sync.WaitGroup{} wg.Add(1) - m := make(chan *proto.StatsEntity) + m := make(chan *metrics.StatsEntityWrapper) go n.Collect(ctx, &wg, m) time.Sleep(100 * time.Millisecond) @@ -97,7 +97,7 @@ func TestNginxWorkerCollector(t *testing.T) { // the prev stats as equal to the initial stats collected // that's ok, but we should test the counter gauge computations again later metricReport := <-m - for _, metric := range metricReport.Simplemetrics { + for _, metric := range metricReport.Data.Simplemetrics { switch metric.Name { case "nginx.workers.cpu.system": assert.Equal(t, float64(0), metric.Value) @@ -137,7 +137,7 @@ func TestNginxWorkerCollector(t *testing.T) { mockBinary.AssertNumberOfCalls(t, "GetChildProcesses", 2) metricReport = <-m - for _, metric := range metricReport.Simplemetrics { + for _, metric := range metricReport.Data.Simplemetrics { switch metric.Name { case "nginx.workers.cpu.system": assert.Equal(t, float64(0), metric.Value) diff --git a/src/core/metrics/sources/swap.go b/src/core/metrics/sources/swap.go index 638917201..027ef0c07 100644 --- a/src/core/metrics/sources/swap.go +++ b/src/core/metrics/sources/swap.go @@ -48,7 +48,7 @@ func NewSwapSource(namespace string, env core.Environment) *Swap { return &Swap{NewMetricSourceLogger(), &namedMetric{namespace, "swap"}, statFunc} } -func (c *Swap) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *Swap) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() swapStats, err := c.statFunc() if err != nil { @@ -73,6 +73,6 @@ func (c *Swap) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto. select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/src/core/metrics/sources/swap_test.go b/src/core/metrics/sources/swap_test.go index e39582e57..54562ed48 100644 --- a/src/core/metrics/sources/swap_test.go +++ b/src/core/metrics/sources/swap_test.go @@ -9,6 +9,7 @@ package sources import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "sync" "testing" @@ -65,11 +66,11 @@ func TestSwapSource_Collect(t *testing.T) { tests := []struct { name string - m chan *proto.StatsEntity + m chan *metrics.StatsEntityWrapper }{ { "basic swap test", - make(chan *proto.StatsEntity, 1), + make(chan *metrics.StatsEntityWrapper, 1), }, } @@ -95,8 +96,8 @@ func TestSwapSource_Collect(t *testing.T) { wg.Wait() statsEntity := <-test.m - assert.Len(tt, statsEntity.Simplemetrics, len(expectedMetrics)) - for _, metric := range statsEntity.Simplemetrics { + assert.Len(tt, statsEntity.Data.Simplemetrics, len(expectedMetrics)) + for _, metric := range statsEntity.Data.Simplemetrics { assert.Contains(tt, expectedMetrics, metric.Name) assert.Equal(t, expectedMetrics[metric.Name], metric.Value) } diff --git a/src/extensions/prometheus-metrics/prometheus_exporter.go b/src/extensions/prometheus-metrics/prometheus_exporter.go index 7215a7dbf..313611bbc 100644 --- a/src/extensions/prometheus-metrics/prometheus_exporter.go +++ b/src/extensions/prometheus-metrics/prometheus_exporter.go @@ -10,19 +10,22 @@ import ( ) type Exporter struct { - latestMetricReport *proto.MetricsReport + latestMetricReports *metrics.MetricsReportBundle } func NewExporter(report *proto.MetricsReport) *Exporter { - return &Exporter{latestMetricReport: report} + return &Exporter{latestMetricReports: &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{report}}} } -func (e *Exporter) SetLatestMetricReport(latest *proto.MetricsReport) { - e.latestMetricReport = latest +func (e *Exporter) SetLatestMetricReport(latest *metrics.MetricsReportBundle) { + e.latestMetricReports = latest } -func (e *Exporter) GetLatestMetricReport() *proto.MetricsReport { - return e.latestMetricReport +func (e *Exporter) GetLatestMetricReports() (reports []*proto.MetricsReport) { + for _, report := range e.latestMetricReports.Data { + reports = append(reports, report) + } + return } func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { @@ -40,9 +43,11 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { } func (e *Exporter) Collect(ch chan<- prometheus.Metric) { - for _, statsEntity := range e.latestMetricReport.Data { - for _, metric := range statsEntity.Simplemetrics { - ch <- createPrometheusMetric(metric, statsEntity.GetDimensions()) + for _, report := range e.latestMetricReports.Data { + for _, statsEntity := range report.Data { + for _, metric := range statsEntity.Simplemetrics { + ch <- createPrometheusMetric(metric, statsEntity.GetDimensions()) + } } } } diff --git a/src/extensions/prometheus-metrics/prometheus_exporter_test.go b/src/extensions/prometheus-metrics/prometheus_exporter_test.go index 3d53a1ee2..cdacb3965 100644 --- a/src/extensions/prometheus-metrics/prometheus_exporter_test.go +++ b/src/extensions/prometheus-metrics/prometheus_exporter_test.go @@ -1,6 +1,7 @@ package prometheus_metrics import ( + "github.com/nginx/agent/v2/src/core/metrics" "testing" "github.com/nginx/agent/sdk/v2/proto" @@ -11,14 +12,20 @@ import ( func TestExporter(t *testing.T) { metricReport1 := &proto.MetricsReport{Meta: &proto.Metadata{MessageId: "123"}} metricReport2 := &proto.MetricsReport{Meta: &proto.Metadata{MessageId: "456"}} + metricReport3 := &proto.MetricsReport{Type: proto.MetricsReport_CACHE_ZONE, Meta: &proto.Metadata{MessageId: "789"}} exporter := NewExporter(metricReport1) - assert.Equal(t, metricReport1, exporter.GetLatestMetricReport()) + assert.Equal(t, metricReport1, exporter.GetLatestMetricReports()[0]) - exporter.SetLatestMetricReport(metricReport2) + exporter.SetLatestMetricReport(&metrics.MetricsReportBundle{Data: []*proto.MetricsReport{metricReport2}}) - assert.Equal(t, metricReport2, exporter.GetLatestMetricReport()) + assert.Equal(t, metricReport2, exporter.GetLatestMetricReports()[0]) + + exporter.SetLatestMetricReport(&metrics.MetricsReportBundle{Data: []*proto.MetricsReport{metricReport2, metricReport3}}) + + assert.Equal(t, metricReport2, exporter.GetLatestMetricReports()[0]) + assert.Equal(t, metricReport3, exporter.GetLatestMetricReports()[1]) } func TestExporter_convertMetricNameToPrometheusFormat(t *testing.T) { diff --git a/src/plugins/agent_api.go b/src/plugins/agent_api.go index a3a626ddd..26f25bbf5 100644 --- a/src/plugins/agent_api.go +++ b/src/plugins/agent_api.go @@ -14,6 +14,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/nginx/agent/v2/src/core/metrics" "io" "net/http" "os" @@ -165,10 +166,10 @@ func (a *AgentAPI) Process(message *core.Message) { } case core.MetricReport: switch response := message.Data().(type) { - case *proto.MetricsReport: + case *metrics.MetricsReportBundle: a.exporter.SetLatestMetricReport(response) default: - log.Warnf("Unknown MetricReport type: %T(%v)", message.Data(), message.Data()) + log.Warnf("Unknown MetricReportBundle type: %T(%v)", message.Data(), message.Data()) } case core.NginxConfigValidationPending, core.NginxConfigApplyFailed, core.NginxConfigApplySucceeded: switch response := message.Data().(type) { diff --git a/src/plugins/agent_api_test.go b/src/plugins/agent_api_test.go index 4a74a673e..b2e2bb3c1 100644 --- a/src/plugins/agent_api_test.go +++ b/src/plugins/agent_api_test.go @@ -14,6 +14,7 @@ import ( "crypto/x509" "encoding/json" "fmt" + "github.com/nginx/agent/v2/src/core/metrics" "io" "io/fs" "io/ioutil" @@ -414,16 +415,17 @@ func TestProcess_metricReport(t *testing.T) { mockNginxBinary := tutils.NewMockNginxBinary() metricReport := &proto.MetricsReport{Meta: &proto.Metadata{MessageId: "123"}} + metricReportBundle := &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{metricReport}} agentAPI := NewAgentAPI(conf, mockEnvironment, mockNginxBinary) // Check that latest metric report isn't set - assert.NotEqual(t, metricReport, agentAPI.exporter.GetLatestMetricReport()) + assert.NotEqual(t, metricReport, agentAPI.exporter.GetLatestMetricReports()[0]) - agentAPI.Process(core.NewMessage(core.MetricReport, metricReport)) + agentAPI.Process(core.NewMessage(core.MetricReport, metricReportBundle)) // Check that latest metric report matches the report that was processed - assert.Equal(t, metricReport, agentAPI.exporter.GetLatestMetricReport()) + assert.Equal(t, metricReport, agentAPI.exporter.GetLatestMetricReports()[0]) } func TestMtls_forApi(t *testing.T) { diff --git a/src/plugins/metrics.go b/src/plugins/metrics.go index 118d3aa3e..8895f37cf 100644 --- a/src/plugins/metrics.go +++ b/src/plugins/metrics.go @@ -12,12 +12,10 @@ import ( "sync" "time" - "github.com/gogo/protobuf/types" log "github.com/sirupsen/logrus" "go.uber.org/atomic" "github.com/nginx/agent/sdk/v2" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" @@ -32,7 +30,7 @@ type Metrics struct { ticker *time.Ticker interval time.Duration collectors []metrics.Collector - buf chan *proto.StatsEntity + buf chan *metrics.StatsEntityWrapper errors chan error collectorConfigsMap map[string]*metrics.NginxCollectorConfig ctx context.Context @@ -53,7 +51,7 @@ func NewMetrics(config *config.Config, env core.Environment, binary core.NginxBi collectorsUpdate: atomic.NewBool(false), ticker: time.NewTicker(config.AgentMetrics.CollectionInterval), interval: config.AgentMetrics.CollectionInterval, - buf: make(chan *proto.StatsEntity, 4096), + buf: make(chan *metrics.StatsEntityWrapper, 4096), errors: make(chan error), collectorConfigsMap: collectorConfigsMap, wg: sync.WaitGroup{}, @@ -184,26 +182,22 @@ func (m *Metrics) metricsGoroutine() { } return case <-m.ticker.C: - report := &proto.MetricsReport{ - Meta: &proto.Metadata{ - Timestamp: types.TimestampNow(), - }, - Type: proto.MetricsReport_SYSTEM, - Data: m.collectStats(), + stats := m.collectStats() + if bundle := metrics.GenerateMetricsReportBundle(stats); bundle != nil { + m.pipeline.Process(core.NewMessage(core.MetricReport, bundle)) } - - m.pipeline.Process(core.NewMessage(core.MetricReport, report)) if m.collectorsUpdate.Load() { m.ticker = time.NewTicker(m.conf.AgentMetrics.CollectionInterval) m.collectorsUpdate.Store(false) } + case err := <-m.errors: log.Errorf("Error in metricsGoroutine %v", err) } } } -func (m *Metrics) collectStats() (stats []*proto.StatsEntity) { +func (m *Metrics) collectStats() (stats []*metrics.StatsEntityWrapper) { // setups a collect duration of half-time of the poll interval ctx, cancel := context.WithTimeout(m.ctx, m.interval/2) defer cancel() @@ -220,6 +214,7 @@ func (m *Metrics) collectStats() (stats []*proto.StatsEntity) { } // wait until all the collection go routines are done, which either context timeout or exit wg.Wait() + for len(m.buf) > 0 { // drain the buf, since our sources/collectors are all done, we can rely on buffer length select { @@ -233,8 +228,9 @@ func (m *Metrics) collectStats() (stats []*proto.StatsEntity) { stats = append(stats, stat) } } + log.Debugf("collected %d entries in %s (ctx error=%t)", len(stats), time.Since(start), ctx.Err() != nil) - return stats + return } func (m *Metrics) registerStatsSources() { diff --git a/src/plugins/metrics_sender.go b/src/plugins/metrics_sender.go index 0b47761c3..d982ee26c 100644 --- a/src/plugins/metrics_sender.go +++ b/src/plugins/metrics_sender.go @@ -79,9 +79,7 @@ func (r *MetricsSender) Process(msg *core.Message) { err := r.reporter.Send(r.ctx, message) if err != nil { - log.Errorf("Failed to send MetricsReport: %v, data: %+v", err, report) - } else { - log.Tracef("MetricsReport sent, %v", report) + log.Errorf("Failed to send MetricsReport: %v", err) } case *models.EventReport: err := r.reporter.Send(r.ctx, client.MessageFromEvents(report)) diff --git a/src/plugins/metrics_throlling.go b/src/plugins/metrics_throlling.go index 373e8950c..1b046762c 100644 --- a/src/plugins/metrics_throlling.go +++ b/src/plugins/metrics_throlling.go @@ -9,6 +9,7 @@ package plugins import ( "context" + "github.com/gogo/protobuf/types" "sync" "time" @@ -33,7 +34,7 @@ type MetricsThrottle struct { reportsReady *atomic.Bool collectorsUpdate *atomic.Bool metricsAggregation bool - metricsCollections metrics.Collections + metricsCollections map[proto.MetricsReport_Type]*metrics.Collections ctx context.Context wg sync.WaitGroup mu sync.Mutex @@ -43,10 +44,6 @@ type MetricsThrottle struct { } func NewMetricsThrottle(conf *config.Config, env core.Environment) *MetricsThrottle { - metricsCollections := metrics.Collections{ - Count: 0, - Data: make(map[string]metrics.PerDimension), - } return &MetricsThrottle{ metricBuffer: make([]core.Payload, 0), @@ -55,7 +52,7 @@ func NewMetricsThrottle(conf *config.Config, env core.Environment) *MetricsThrot reportsReady: atomic.NewBool(false), collectorsUpdate: atomic.NewBool(false), metricsAggregation: conf.AgentMetrics.Mode == "aggregated", - metricsCollections: metricsCollections, + metricsCollections: make(map[proto.MetricsReport_Type]*metrics.Collections, 0), wg: sync.WaitGroup{}, env: env, conf: conf, @@ -92,21 +89,42 @@ func (r *MetricsThrottle) Process(msg *core.Message) { return case msg.Exact(core.MetricReport): if r.metricsAggregation { - switch report := msg.Data().(type) { - case *proto.MetricsReport: - r.mu.Lock() - r.metricsCollections = metrics.SaveCollections(r.metricsCollections, report) - r.mu.Unlock() - log.Debug("MetricsThrottle: Metrics collection saved") - r.reportsReady.Store(true) + switch bundle := msg.Data().(type) { + case *metrics.MetricsReportBundle: + if len(bundle.Data) > 0 { + r.mu.Lock() + for _, report := range bundle.Data { + if len(report.Data) > 0 { + if _, ok := r.metricsCollections[report.Type]; !ok { + r.metricsCollections[report.Type] = &metrics.Collections{ + Count: 0, + Data: make(map[string]metrics.PerDimension), + } + } + collection := metrics.SaveCollections(*r.metricsCollections[report.Type], report) + r.metricsCollections[report.Type] = &collection + log.Debugf("MetricsThrottle: Metrics collection saved [Type: %d]", report.Type) + } + } + r.mu.Unlock() + r.reportsReady.Store(true) + } } } else { - r.metricBuffer = append(r.metricBuffer, msg.Data()) + switch bundle := msg.Data().(type) { + case *metrics.MetricsReportBundle: + if len(bundle.Data) > 0 { + for _, report := range bundle.Data { + if len(report.Data) > 0 { + r.metricBuffer = append(r.metricBuffer, report) + } + } + } + } + log.Tracef("MetricsThrottle buffer size: %d of %d", len(r.metricBuffer), r.BulkSize) if len(r.metricBuffer) >= r.BulkSize { log.Info("MetricsThrottle buffer flush") - r.messagePipeline.Process( - core.NewMessage(core.CommMetrics, r.metricBuffer), - ) + r.messagePipeline.Process(core.NewMessage(core.CommMetrics, r.metricBuffer)) r.metricBuffer = make([]core.Payload, 0) } } @@ -134,10 +152,10 @@ func (r *MetricsThrottle) metricsReportGoroutine(ctx context.Context, wg *sync.W } return case <-r.ticker.C: - aggregatedReport := r.getAggregatedReport() - r.messagePipeline.Process( - core.NewMessage(core.CommMetrics, []core.Payload{aggregatedReport}), - ) + reports := r.getAggregatedReports() + if len(reports) > 0 { + r.messagePipeline.Process(core.NewMessage(core.CommMetrics, reports)) + } if r.collectorsUpdate.Load() { r.BulkSize = r.conf.AgentMetrics.BulkSize r.metricsAggregation = r.conf.AgentMetrics.Mode == "aggregated" @@ -167,13 +185,23 @@ func (r *MetricsThrottle) syncAgentConfigChange() { r.conf = conf } -func (r *MetricsThrottle) getAggregatedReport() *proto.MetricsReport { +func (r *MetricsThrottle) getAggregatedReports() (reports []core.Payload) { r.mu.Lock() defer r.mu.Unlock() - report := metrics.GenerateMetricsReport(r.metricsCollections) - r.metricsCollections = metrics.Collections{ - Count: 0, - Data: make(map[string]metrics.PerDimension), + + for reportType, collection := range r.metricsCollections { + reports = append(reports, &proto.MetricsReport{ + Meta: &proto.Metadata{ + Timestamp: types.TimestampNow(), + }, + Type: reportType, + Data: metrics.GenerateMetrics(*collection), + }) + r.metricsCollections[reportType] = &metrics.Collections{ + Count: 0, + Data: make(map[string]metrics.PerDimension), + } } - return report + + return } diff --git a/src/plugins/metrics_throlling_test.go b/src/plugins/metrics_throlling_test.go index e7853c96c..4577e5047 100644 --- a/src/plugins/metrics_throlling_test.go +++ b/src/plugins/metrics_throlling_test.go @@ -9,6 +9,7 @@ package plugins import ( "context" + "github.com/nginx/agent/v2/src/core/metrics" "testing" "github.com/nginx/agent/sdk/v2/proto" @@ -30,7 +31,7 @@ func TestMetricsThrottle_Process(t *testing.T) { { name: "not enough metrics", msgs: []*core.Message{ - core.NewMessage(core.MetricReport, &proto.MetricsReport{}), + core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{{}}}), }, msgTopics: []string{ core.MetricReport, @@ -38,11 +39,11 @@ func TestMetricsThrottle_Process(t *testing.T) { config: tutils.GetMockAgentConfig(), }, { - name: "flush buffer of metrics streaming", + name: "flush buffer of metrics streaming - nonempty reports", msgs: []*core.Message{ - core.NewMessage(core.MetricReport, &proto.MetricsReport{}), - core.NewMessage(core.MetricReport, &proto.MetricsReport{}), - core.NewMessage(core.MetricReport, &proto.MetricsReport{}), + core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{{Data: []*proto.StatsEntity{&proto.StatsEntity{}}}}}), + core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{{Data: []*proto.StatsEntity{&proto.StatsEntity{}}}}}), + core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{{Data: []*proto.StatsEntity{&proto.StatsEntity{}}}}}), }, msgTopics: []string{ core.MetricReport, @@ -63,12 +64,35 @@ func TestMetricsThrottle_Process(t *testing.T) { }, }, }, + { + name: "flush buffer of metrics streaming - empty reports", + msgs: []*core.Message{ + core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{{}}}), + core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{{}}}), + core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{{}}}), + }, + msgTopics: []string{ + core.MetricReport, + core.MetricReport, + core.MetricReport, + }, + config: &config.Config{ + ClientID: "12345", + Tags: tutils.InitialConfTags, + AgentMetrics: config.AgentMetrics{ + BulkSize: 1, + ReportInterval: 5, + CollectionInterval: 1, + Mode: "streaming", + }, + }, + }, { name: "flush buffer of metrics", msgs: []*core.Message{ - core.NewMessage(core.MetricReport, &proto.MetricsReport{}), - core.NewMessage(core.MetricReport, &proto.MetricsReport{}), - core.NewMessage(core.MetricReport, &proto.MetricsReport{}), + core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{{}}}), + core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{{}}}), + core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{{}}}), }, msgTopics: []string{ core.MetricReport, diff --git a/test/component/agent_api_test.go b/test/component/agent_api_test.go index fab531251..b55794453 100644 --- a/test/component/agent_api_test.go +++ b/test/component/agent_api_test.go @@ -3,6 +3,7 @@ package component import ( "context" "fmt" + "github.com/nginx/agent/v2/src/core/metrics" "net/http" "strconv" "strings" @@ -135,35 +136,60 @@ func TestMetrics(t *testing.T) { agentAPI := plugins.NewAgentAPI(conf, mockEnvironment, mockNginxBinary) agentAPI.Init(core.NewMockMessagePipe(context.TODO())) - agentAPI.Process(core.NewMessage(core.MetricReport, &proto.MetricsReport{ - Meta: &proto.Metadata{ - MessageId: "123", - }, - Data: []*proto.StatsEntity{ - { - Dimensions: []*proto.Dimension{ - { - Name: "hostname", - Value: "example.com", + agentAPI.Process(core.NewMessage(core.MetricReport, &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{ + { + Type: proto.MetricsReport_SYSTEM, + Meta: &proto.Metadata{ + MessageId: "123", + }, + Data: []*proto.StatsEntity{ + { + Dimensions: []*proto.Dimension{ + { + Name: "hostname", + Value: "example.com", + }, + { + Name: "system.tags", + Value: "", + }, }, - { - Name: "system.tags", - Value: "", + Simplemetrics: []*proto.SimpleMetric{ + { + Name: "system.cpu.idle", + Value: 12, + }, }, }, - Simplemetrics: []*proto.SimpleMetric{ - { - Name: "system.cpu.idle", - Value: 12, + }, + }, + { + Type: proto.MetricsReport_INSTANCE, + Meta: &proto.Metadata{ + MessageId: "456", + }, + Data: []*proto.StatsEntity{ + { + Dimensions: []*proto.Dimension{ + { + Name: "hostname", + Value: "example.com", + }, + { + Name: "system.tags", + Value: "", + }, }, - { - Name: "nginx.workers.count", - Value: 6, + Simplemetrics: []*proto.SimpleMetric{ + { + Name: "nginx.workers.count", + Value: 6, + }, }, }, }, }, - })) + }})) client := resty.New() diff --git a/test/integration/api/api_test.go b/test/integration/api/api_test.go index 449243440..21048395c 100644 --- a/test/integration/api/api_test.go +++ b/test/integration/api/api_test.go @@ -73,7 +73,6 @@ func TestAPI_Metrics(t *testing.T) { for _, m := range metrics { metric := strings.Split(m, " ") - switch { case strings.Contains(metric[0], "system_cpu_system"): value, _ := strconv.ParseFloat(metric[1], 64) diff --git a/test/integration/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go b/test/integration/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go index 06fe5c328..46e46326b 100644 --- a/test/integration/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go +++ b/test/integration/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go @@ -157,7 +157,7 @@ func (r *metricReporter) Send(ctx context.Context, message Message) error { return r.handleGrpcError("Metric Reporter Channel Send", err) } - log.Tracef("MetricReporter sent metrics report %v", report) + log.Tracef("MetricReporter sent metrics report [Type: %d] %+v", report.Type, report) return nil }) diff --git a/test/integration/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go b/test/integration/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go index ee8c82794..e6ad339ef 100644 --- a/test/integration/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go +++ b/test/integration/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go @@ -35,18 +35,26 @@ const ( MetricsReport_INSTANCE MetricsReport_Type = 1 // Agent metric type MetricsReport_AGENT MetricsReport_Type = 2 + // Cache zone metric type + MetricsReport_CACHE_ZONE MetricsReport_Type = 3 + // Upstreams metric type + MetricsReport_UPSTREAMS MetricsReport_Type = 4 ) var MetricsReport_Type_name = map[int32]string{ 0: "SYSTEM", 1: "INSTANCE", 2: "AGENT", + 3: "CACHE_ZONE", + 4: "UPSTREAMS", } var MetricsReport_Type_value = map[string]int32{ - "SYSTEM": 0, - "INSTANCE": 1, - "AGENT": 2, + "SYSTEM": 0, + "INSTANCE": 1, + "AGENT": 2, + "CACHE_ZONE": 3, + "UPSTREAMS": 4, } func (x MetricsReport_Type) String() string { @@ -318,36 +326,38 @@ func init() { func init() { proto.RegisterFile("metrics.proto", fileDescriptor_6039342a2ba47b72) } var fileDescriptor_6039342a2ba47b72 = []byte{ - // 460 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x52, 0x41, 0x6f, 0xd3, 0x30, - 0x14, 0xc6, 0x6d, 0x36, 0x9a, 0xd7, 0x76, 0x2a, 0x3e, 0x55, 0xd5, 0x68, 0xaa, 0x1e, 0x50, 0x11, - 0x92, 0x23, 0x15, 0x21, 0x10, 0x88, 0xc3, 0xc2, 0x2a, 0xc4, 0xa4, 0xf4, 0xe0, 0xe4, 0x32, 0x6e, - 0x5e, 0xeb, 0x85, 0x68, 0x75, 0x1c, 0xd5, 0xee, 0x44, 0x6f, 0xfc, 0x3c, 0x8e, 0xfc, 0x82, 0x0a, - 0xf5, 0xd8, 0xff, 0x80, 0x84, 0x62, 0xa7, 0x5b, 0x27, 0xc2, 0x61, 0x97, 0x67, 0x3f, 0xeb, 0x7d, - 0xdf, 0x7b, 0xef, 0xfb, 0x0c, 0x6d, 0xc1, 0xf5, 0x32, 0x9d, 0x29, 0x92, 0x2f, 0xa5, 0x96, 0x18, - 0x5f, 0xbf, 0x21, 0x59, 0x92, 0x66, 0xdf, 0x09, 0x4b, 0x78, 0xa6, 0x89, 0x9a, 0xdf, 0xf4, 0x5a, - 0x33, 0x29, 0x84, 0xcc, 0x6c, 0x45, 0x0f, 0x12, 0x99, 0xc8, 0xf2, 0xee, 0x25, 0x52, 0x26, 0x0b, - 0xee, 0x9b, 0xec, 0x6a, 0x75, 0xed, 0xeb, 0x54, 0x70, 0xa5, 0x99, 0xc8, 0x6d, 0xc1, 0xf0, 0x0f, - 0x82, 0x76, 0x68, 0x1b, 0x50, 0x9e, 0xcb, 0xa5, 0xc6, 0xef, 0xc1, 0x11, 0x5c, 0xb3, 0x2e, 0x1a, - 0xa0, 0x51, 0x73, 0x7c, 0x4a, 0xfe, 0xed, 0x47, 0x42, 0xae, 0xd9, 0x9c, 0x69, 0x16, 0x34, 0x76, - 0x1b, 0xcf, 0x54, 0x53, 0x13, 0xf1, 0x39, 0x38, 0x7a, 0x9d, 0xf3, 0x6e, 0x6d, 0x80, 0x46, 0x27, - 0xe3, 0x17, 0xff, 0xc1, 0xde, 0x37, 0x23, 0xf1, 0x3a, 0xe7, 0x96, 0xa5, 0xc0, 0x51, 0x13, 0xf1, - 0x47, 0x70, 0x0a, 0xf6, 0x6e, 0x7d, 0x50, 0x1f, 0x35, 0xc7, 0x5e, 0x15, 0x4b, 0xa4, 0x99, 0x56, - 0x93, 0x4c, 0xa7, 0x7a, 0x6d, 0xe1, 0x05, 0x80, 0x9a, 0x38, 0x7c, 0x05, 0x4e, 0x41, 0x8b, 0x01, - 0x8e, 0xa3, 0xcb, 0x28, 0x9e, 0x84, 0x9d, 0x27, 0xb8, 0x05, 0x8d, 0x2f, 0xd3, 0x28, 0x3e, 0x9b, - 0x7e, 0x9a, 0x74, 0x10, 0x76, 0xe1, 0xe8, 0xec, 0xf3, 0x64, 0x1a, 0x77, 0x6a, 0xc3, 0x10, 0x5a, - 0x51, 0x2a, 0xf2, 0x05, 0xb7, 0x73, 0xe1, 0x53, 0x70, 0x32, 0x26, 0xb8, 0xd9, 0xde, 0xb5, 0xd4, - 0x45, 0x4e, 0x4d, 0xc4, 0x1e, 0x1c, 0xdd, 0xb2, 0xc5, 0xca, 0x2e, 0x88, 0x02, 0x77, 0xb7, 0xf1, - 0xec, 0x03, 0xb5, 0xc7, 0xf0, 0x02, 0xdc, 0xf3, 0x54, 0xf0, 0x4c, 0xa5, 0x32, 0x7b, 0x0c, 0x97, - 0x5b, 0xc1, 0xf5, 0xa3, 0x06, 0xcd, 0x83, 0x3d, 0xf1, 0x3b, 0x70, 0xef, 0xdc, 0x2b, 0xdd, 0xe9, - 0x11, 0xeb, 0x2f, 0xd9, 0xfb, 0x4b, 0xe2, 0x7d, 0x05, 0xbd, 0x2f, 0xc6, 0x21, 0xc0, 0x7c, 0x3f, - 0x95, 0x2a, 0x65, 0x7d, 0x5e, 0x25, 0xeb, 0xdd, 0xec, 0xc1, 0xc9, 0x6e, 0xe3, 0x1d, 0x80, 0xe8, - 0xc1, 0x1d, 0x5f, 0x42, 0x5b, 0x19, 0xcd, 0xca, 0x9f, 0xd9, 0x75, 0x0c, 0xe3, 0xa0, 0xd2, 0xa8, - 0x03, 0x71, 0x83, 0x67, 0xbb, 0x8d, 0xf7, 0x10, 0x4a, 0x1f, 0xa6, 0x17, 0x4e, 0xa3, 0xd6, 0xa9, - 0xd3, 0xa7, 0x65, 0x1a, 0xbc, 0xfd, 0xb9, 0xed, 0xa3, 0x5f, 0xdb, 0x3e, 0xfa, 0xbd, 0xed, 0xa3, - 0xaf, 0x2f, 0x93, 0x54, 0x7f, 0x5b, 0x5d, 0x91, 0x99, 0x14, 0xbe, 0xe9, 0xe3, 0x9b, 0x3e, 0xbe, - 0x9a, 0xdf, 0xf8, 0xb7, 0x63, 0xfb, 0xc7, 0x3f, 0x58, 0x25, 0x8e, 0xcd, 0xf1, 0xfa, 0x6f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xe6, 0xcd, 0x1e, 0xb5, 0x3d, 0x03, 0x00, 0x00, + // 487 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xc5, 0x8e, 0x5b, 0xe2, 0xc9, 0x87, 0xcc, 0x9e, 0xa2, 0xa8, 0xc4, 0x51, 0x0e, 0x28, 0x5c, + 0xd6, 0x52, 0x10, 0x02, 0x81, 0x38, 0xc4, 0xa9, 0x05, 0x44, 0x72, 0x40, 0x6b, 0x73, 0x68, 0x2f, + 0xc8, 0x4d, 0xb6, 0xc6, 0x6a, 0xd6, 0x6b, 0xc5, 0x9b, 0x8a, 0xdc, 0xf8, 0x21, 0xfc, 0x20, 0x8e, + 0xfc, 0x82, 0x08, 0xe5, 0x98, 0x5f, 0x81, 0xbc, 0xeb, 0xb4, 0x89, 0x08, 0x87, 0x5e, 0x66, 0x77, + 0xac, 0x79, 0xef, 0x8d, 0xdf, 0xb3, 0xa1, 0xc1, 0xa8, 0x58, 0x24, 0xd3, 0x1c, 0x67, 0x0b, 0x2e, + 0x38, 0x42, 0xd7, 0x2f, 0x71, 0x1a, 0x27, 0xe9, 0x77, 0x1c, 0xc5, 0x34, 0x15, 0x38, 0x9f, 0xdd, + 0xb4, 0xeb, 0x53, 0xce, 0x18, 0x4f, 0xd5, 0x44, 0x1b, 0x62, 0x1e, 0xf3, 0xf2, 0x6e, 0xc7, 0x9c, + 0xc7, 0x73, 0xea, 0xc8, 0xee, 0x6a, 0x79, 0xed, 0x88, 0x84, 0xd1, 0x5c, 0x44, 0x2c, 0x53, 0x03, + 0xbd, 0x9f, 0x3a, 0x34, 0x7c, 0x25, 0x40, 0x68, 0xc6, 0x17, 0x02, 0xbd, 0x01, 0x83, 0x51, 0x11, + 0xb5, 0xb4, 0xae, 0xd6, 0xaf, 0x0d, 0xce, 0xf0, 0xbf, 0x7a, 0xd8, 0xa7, 0x22, 0x9a, 0x45, 0x22, + 0x72, 0xab, 0xdb, 0xb5, 0x2d, 0xa7, 0x89, 0xac, 0xe8, 0x1c, 0x0c, 0xb1, 0xca, 0x68, 0x4b, 0xef, + 0x6a, 0xfd, 0xe6, 0xe0, 0xd9, 0x7f, 0xb0, 0xf7, 0x62, 0x38, 0x5c, 0x65, 0x54, 0xb1, 0x14, 0x38, + 0x22, 0x2b, 0x7a, 0x07, 0x46, 0xc1, 0xde, 0xaa, 0x74, 0x2b, 0xfd, 0xda, 0xc0, 0x3e, 0xc6, 0x12, + 0x88, 0x48, 0xe4, 0x5e, 0x2a, 0x12, 0xb1, 0x52, 0xf0, 0x02, 0x40, 0x64, 0xed, 0x8d, 0xc1, 0x28, + 0x68, 0x11, 0xc0, 0x69, 0x70, 0x11, 0x84, 0x9e, 0x6f, 0x3d, 0x42, 0x75, 0xa8, 0x7e, 0x9c, 0x04, + 0xe1, 0x70, 0x32, 0xf2, 0x2c, 0x0d, 0x99, 0x70, 0x32, 0x7c, 0xef, 0x4d, 0x42, 0x4b, 0x47, 0x4d, + 0x80, 0xd1, 0x70, 0xf4, 0xc1, 0xfb, 0x7a, 0xf9, 0x69, 0xe2, 0x59, 0x15, 0xd4, 0x00, 0xf3, 0xcb, + 0xe7, 0x20, 0x24, 0xde, 0xd0, 0x0f, 0x2c, 0xa3, 0xe7, 0x43, 0x3d, 0x48, 0x58, 0x36, 0xa7, 0x6a, + 0x6d, 0x74, 0x06, 0x46, 0x1a, 0x31, 0x2a, 0xcd, 0x31, 0x95, 0x72, 0xd1, 0x13, 0x59, 0x91, 0x0d, + 0x27, 0xb7, 0xd1, 0x7c, 0xa9, 0xde, 0x5f, 0x73, 0xcd, 0xed, 0xda, 0x56, 0x0f, 0x88, 0x3a, 0x7a, + 0x63, 0x30, 0xcf, 0x13, 0x46, 0xd3, 0x3c, 0xe1, 0xe9, 0x43, 0xb8, 0xcc, 0x23, 0x5c, 0x3f, 0x74, + 0xa8, 0xed, 0xd9, 0x80, 0x5e, 0x83, 0x79, 0x17, 0x6e, 0x19, 0x5e, 0x1b, 0xab, 0xf8, 0xf1, 0x2e, + 0x7e, 0x1c, 0xee, 0x26, 0xc8, 0xfd, 0x30, 0xf2, 0x01, 0x66, 0xbb, 0xad, 0xf2, 0xd2, 0xf5, 0xa7, + 0xc7, 0x5c, 0xbf, 0xdb, 0xdd, 0x6d, 0x6e, 0xd7, 0xf6, 0x1e, 0x88, 0xec, 0xdd, 0xd1, 0x05, 0x34, + 0x72, 0xe9, 0x59, 0xf9, 0xe1, 0xb6, 0x0c, 0xc9, 0xd8, 0x3d, 0x9a, 0xe3, 0x9e, 0xb9, 0xee, 0x93, + 0xed, 0xda, 0x3e, 0x84, 0x92, 0xc3, 0x76, 0x6c, 0x54, 0x75, 0xab, 0x42, 0x1e, 0x97, 0xad, 0xfb, + 0xea, 0xd7, 0xa6, 0xa3, 0xfd, 0xde, 0x74, 0xb4, 0x3f, 0x9b, 0x8e, 0x76, 0xf9, 0x3c, 0x4e, 0xc4, + 0xb7, 0xe5, 0x15, 0x9e, 0x72, 0xe6, 0x48, 0x1d, 0x47, 0xea, 0x38, 0xf9, 0xec, 0xc6, 0xb9, 0x1d, + 0xa8, 0x5f, 0xe0, 0xad, 0x72, 0xe2, 0x54, 0x1e, 0x2f, 0xfe, 0x06, 0x00, 0x00, 0xff, 0xff, 0x87, + 0xa1, 0xae, 0x17, 0x5c, 0x03, 0x00, 0x00, } func (m *MetricsReport) Marshal() (dAtA []byte, err error) { diff --git a/test/integration/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto b/test/integration/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto index e4cba46fd..fa3cf941d 100644 --- a/test/integration/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto +++ b/test/integration/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto @@ -17,6 +17,10 @@ message MetricsReport { INSTANCE = 1; // Agent metric type AGENT = 2; + // Cache zone metric type + CACHE_ZONE = 3; + // Upstreams metric type + UPSTREAMS = 4; } // Provides meta information about the metrics Metadata meta = 1 [(gogoproto.jsontag) = "meta"]; diff --git a/test/performance/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go b/test/performance/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go index 06fe5c328..46e46326b 100644 --- a/test/performance/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go +++ b/test/performance/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go @@ -157,7 +157,7 @@ func (r *metricReporter) Send(ctx context.Context, message Message) error { return r.handleGrpcError("Metric Reporter Channel Send", err) } - log.Tracef("MetricReporter sent metrics report %v", report) + log.Tracef("MetricReporter sent metrics report [Type: %d] %+v", report.Type, report) return nil }) diff --git a/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go b/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go index ee8c82794..e6ad339ef 100644 --- a/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go +++ b/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go @@ -35,18 +35,26 @@ const ( MetricsReport_INSTANCE MetricsReport_Type = 1 // Agent metric type MetricsReport_AGENT MetricsReport_Type = 2 + // Cache zone metric type + MetricsReport_CACHE_ZONE MetricsReport_Type = 3 + // Upstreams metric type + MetricsReport_UPSTREAMS MetricsReport_Type = 4 ) var MetricsReport_Type_name = map[int32]string{ 0: "SYSTEM", 1: "INSTANCE", 2: "AGENT", + 3: "CACHE_ZONE", + 4: "UPSTREAMS", } var MetricsReport_Type_value = map[string]int32{ - "SYSTEM": 0, - "INSTANCE": 1, - "AGENT": 2, + "SYSTEM": 0, + "INSTANCE": 1, + "AGENT": 2, + "CACHE_ZONE": 3, + "UPSTREAMS": 4, } func (x MetricsReport_Type) String() string { @@ -318,36 +326,38 @@ func init() { func init() { proto.RegisterFile("metrics.proto", fileDescriptor_6039342a2ba47b72) } var fileDescriptor_6039342a2ba47b72 = []byte{ - // 460 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x52, 0x41, 0x6f, 0xd3, 0x30, - 0x14, 0xc6, 0x6d, 0x36, 0x9a, 0xd7, 0x76, 0x2a, 0x3e, 0x55, 0xd5, 0x68, 0xaa, 0x1e, 0x50, 0x11, - 0x92, 0x23, 0x15, 0x21, 0x10, 0x88, 0xc3, 0xc2, 0x2a, 0xc4, 0xa4, 0xf4, 0xe0, 0xe4, 0x32, 0x6e, - 0x5e, 0xeb, 0x85, 0x68, 0x75, 0x1c, 0xd5, 0xee, 0x44, 0x6f, 0xfc, 0x3c, 0x8e, 0xfc, 0x82, 0x0a, - 0xf5, 0xd8, 0xff, 0x80, 0x84, 0x62, 0xa7, 0x5b, 0x27, 0xc2, 0x61, 0x97, 0x67, 0x3f, 0xeb, 0x7d, - 0xdf, 0x7b, 0xef, 0xfb, 0x0c, 0x6d, 0xc1, 0xf5, 0x32, 0x9d, 0x29, 0x92, 0x2f, 0xa5, 0x96, 0x18, - 0x5f, 0xbf, 0x21, 0x59, 0x92, 0x66, 0xdf, 0x09, 0x4b, 0x78, 0xa6, 0x89, 0x9a, 0xdf, 0xf4, 0x5a, - 0x33, 0x29, 0x84, 0xcc, 0x6c, 0x45, 0x0f, 0x12, 0x99, 0xc8, 0xf2, 0xee, 0x25, 0x52, 0x26, 0x0b, - 0xee, 0x9b, 0xec, 0x6a, 0x75, 0xed, 0xeb, 0x54, 0x70, 0xa5, 0x99, 0xc8, 0x6d, 0xc1, 0xf0, 0x0f, - 0x82, 0x76, 0x68, 0x1b, 0x50, 0x9e, 0xcb, 0xa5, 0xc6, 0xef, 0xc1, 0x11, 0x5c, 0xb3, 0x2e, 0x1a, - 0xa0, 0x51, 0x73, 0x7c, 0x4a, 0xfe, 0xed, 0x47, 0x42, 0xae, 0xd9, 0x9c, 0x69, 0x16, 0x34, 0x76, - 0x1b, 0xcf, 0x54, 0x53, 0x13, 0xf1, 0x39, 0x38, 0x7a, 0x9d, 0xf3, 0x6e, 0x6d, 0x80, 0x46, 0x27, - 0xe3, 0x17, 0xff, 0xc1, 0xde, 0x37, 0x23, 0xf1, 0x3a, 0xe7, 0x96, 0xa5, 0xc0, 0x51, 0x13, 0xf1, - 0x47, 0x70, 0x0a, 0xf6, 0x6e, 0x7d, 0x50, 0x1f, 0x35, 0xc7, 0x5e, 0x15, 0x4b, 0xa4, 0x99, 0x56, - 0x93, 0x4c, 0xa7, 0x7a, 0x6d, 0xe1, 0x05, 0x80, 0x9a, 0x38, 0x7c, 0x05, 0x4e, 0x41, 0x8b, 0x01, - 0x8e, 0xa3, 0xcb, 0x28, 0x9e, 0x84, 0x9d, 0x27, 0xb8, 0x05, 0x8d, 0x2f, 0xd3, 0x28, 0x3e, 0x9b, - 0x7e, 0x9a, 0x74, 0x10, 0x76, 0xe1, 0xe8, 0xec, 0xf3, 0x64, 0x1a, 0x77, 0x6a, 0xc3, 0x10, 0x5a, - 0x51, 0x2a, 0xf2, 0x05, 0xb7, 0x73, 0xe1, 0x53, 0x70, 0x32, 0x26, 0xb8, 0xd9, 0xde, 0xb5, 0xd4, - 0x45, 0x4e, 0x4d, 0xc4, 0x1e, 0x1c, 0xdd, 0xb2, 0xc5, 0xca, 0x2e, 0x88, 0x02, 0x77, 0xb7, 0xf1, - 0xec, 0x03, 0xb5, 0xc7, 0xf0, 0x02, 0xdc, 0xf3, 0x54, 0xf0, 0x4c, 0xa5, 0x32, 0x7b, 0x0c, 0x97, - 0x5b, 0xc1, 0xf5, 0xa3, 0x06, 0xcd, 0x83, 0x3d, 0xf1, 0x3b, 0x70, 0xef, 0xdc, 0x2b, 0xdd, 0xe9, - 0x11, 0xeb, 0x2f, 0xd9, 0xfb, 0x4b, 0xe2, 0x7d, 0x05, 0xbd, 0x2f, 0xc6, 0x21, 0xc0, 0x7c, 0x3f, - 0x95, 0x2a, 0x65, 0x7d, 0x5e, 0x25, 0xeb, 0xdd, 0xec, 0xc1, 0xc9, 0x6e, 0xe3, 0x1d, 0x80, 0xe8, - 0xc1, 0x1d, 0x5f, 0x42, 0x5b, 0x19, 0xcd, 0xca, 0x9f, 0xd9, 0x75, 0x0c, 0xe3, 0xa0, 0xd2, 0xa8, - 0x03, 0x71, 0x83, 0x67, 0xbb, 0x8d, 0xf7, 0x10, 0x4a, 0x1f, 0xa6, 0x17, 0x4e, 0xa3, 0xd6, 0xa9, - 0xd3, 0xa7, 0x65, 0x1a, 0xbc, 0xfd, 0xb9, 0xed, 0xa3, 0x5f, 0xdb, 0x3e, 0xfa, 0xbd, 0xed, 0xa3, - 0xaf, 0x2f, 0x93, 0x54, 0x7f, 0x5b, 0x5d, 0x91, 0x99, 0x14, 0xbe, 0xe9, 0xe3, 0x9b, 0x3e, 0xbe, - 0x9a, 0xdf, 0xf8, 0xb7, 0x63, 0xfb, 0xc7, 0x3f, 0x58, 0x25, 0x8e, 0xcd, 0xf1, 0xfa, 0x6f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xe6, 0xcd, 0x1e, 0xb5, 0x3d, 0x03, 0x00, 0x00, + // 487 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xc5, 0x8e, 0x5b, 0xe2, 0xc9, 0x87, 0xcc, 0x9e, 0xa2, 0xa8, 0xc4, 0x51, 0x0e, 0x28, 0x5c, + 0xd6, 0x52, 0x10, 0x02, 0x81, 0x38, 0xc4, 0xa9, 0x05, 0x44, 0x72, 0x40, 0x6b, 0x73, 0x68, 0x2f, + 0xc8, 0x4d, 0xb6, 0xc6, 0x6a, 0xd6, 0x6b, 0xc5, 0x9b, 0x8a, 0xdc, 0xf8, 0x21, 0xfc, 0x20, 0x8e, + 0xfc, 0x82, 0x08, 0xe5, 0x98, 0x5f, 0x81, 0xbc, 0xeb, 0xb4, 0x89, 0x08, 0x87, 0x5e, 0x66, 0x77, + 0xac, 0x79, 0xef, 0x8d, 0xdf, 0xb3, 0xa1, 0xc1, 0xa8, 0x58, 0x24, 0xd3, 0x1c, 0x67, 0x0b, 0x2e, + 0x38, 0x42, 0xd7, 0x2f, 0x71, 0x1a, 0x27, 0xe9, 0x77, 0x1c, 0xc5, 0x34, 0x15, 0x38, 0x9f, 0xdd, + 0xb4, 0xeb, 0x53, 0xce, 0x18, 0x4f, 0xd5, 0x44, 0x1b, 0x62, 0x1e, 0xf3, 0xf2, 0x6e, 0xc7, 0x9c, + 0xc7, 0x73, 0xea, 0xc8, 0xee, 0x6a, 0x79, 0xed, 0x88, 0x84, 0xd1, 0x5c, 0x44, 0x2c, 0x53, 0x03, + 0xbd, 0x9f, 0x3a, 0x34, 0x7c, 0x25, 0x40, 0x68, 0xc6, 0x17, 0x02, 0xbd, 0x01, 0x83, 0x51, 0x11, + 0xb5, 0xb4, 0xae, 0xd6, 0xaf, 0x0d, 0xce, 0xf0, 0xbf, 0x7a, 0xd8, 0xa7, 0x22, 0x9a, 0x45, 0x22, + 0x72, 0xab, 0xdb, 0xb5, 0x2d, 0xa7, 0x89, 0xac, 0xe8, 0x1c, 0x0c, 0xb1, 0xca, 0x68, 0x4b, 0xef, + 0x6a, 0xfd, 0xe6, 0xe0, 0xd9, 0x7f, 0xb0, 0xf7, 0x62, 0x38, 0x5c, 0x65, 0x54, 0xb1, 0x14, 0x38, + 0x22, 0x2b, 0x7a, 0x07, 0x46, 0xc1, 0xde, 0xaa, 0x74, 0x2b, 0xfd, 0xda, 0xc0, 0x3e, 0xc6, 0x12, + 0x88, 0x48, 0xe4, 0x5e, 0x2a, 0x12, 0xb1, 0x52, 0xf0, 0x02, 0x40, 0x64, 0xed, 0x8d, 0xc1, 0x28, + 0x68, 0x11, 0xc0, 0x69, 0x70, 0x11, 0x84, 0x9e, 0x6f, 0x3d, 0x42, 0x75, 0xa8, 0x7e, 0x9c, 0x04, + 0xe1, 0x70, 0x32, 0xf2, 0x2c, 0x0d, 0x99, 0x70, 0x32, 0x7c, 0xef, 0x4d, 0x42, 0x4b, 0x47, 0x4d, + 0x80, 0xd1, 0x70, 0xf4, 0xc1, 0xfb, 0x7a, 0xf9, 0x69, 0xe2, 0x59, 0x15, 0xd4, 0x00, 0xf3, 0xcb, + 0xe7, 0x20, 0x24, 0xde, 0xd0, 0x0f, 0x2c, 0xa3, 0xe7, 0x43, 0x3d, 0x48, 0x58, 0x36, 0xa7, 0x6a, + 0x6d, 0x74, 0x06, 0x46, 0x1a, 0x31, 0x2a, 0xcd, 0x31, 0x95, 0x72, 0xd1, 0x13, 0x59, 0x91, 0x0d, + 0x27, 0xb7, 0xd1, 0x7c, 0xa9, 0xde, 0x5f, 0x73, 0xcd, 0xed, 0xda, 0x56, 0x0f, 0x88, 0x3a, 0x7a, + 0x63, 0x30, 0xcf, 0x13, 0x46, 0xd3, 0x3c, 0xe1, 0xe9, 0x43, 0xb8, 0xcc, 0x23, 0x5c, 0x3f, 0x74, + 0xa8, 0xed, 0xd9, 0x80, 0x5e, 0x83, 0x79, 0x17, 0x6e, 0x19, 0x5e, 0x1b, 0xab, 0xf8, 0xf1, 0x2e, + 0x7e, 0x1c, 0xee, 0x26, 0xc8, 0xfd, 0x30, 0xf2, 0x01, 0x66, 0xbb, 0xad, 0xf2, 0xd2, 0xf5, 0xa7, + 0xc7, 0x5c, 0xbf, 0xdb, 0xdd, 0x6d, 0x6e, 0xd7, 0xf6, 0x1e, 0x88, 0xec, 0xdd, 0xd1, 0x05, 0x34, + 0x72, 0xe9, 0x59, 0xf9, 0xe1, 0xb6, 0x0c, 0xc9, 0xd8, 0x3d, 0x9a, 0xe3, 0x9e, 0xb9, 0xee, 0x93, + 0xed, 0xda, 0x3e, 0x84, 0x92, 0xc3, 0x76, 0x6c, 0x54, 0x75, 0xab, 0x42, 0x1e, 0x97, 0xad, 0xfb, + 0xea, 0xd7, 0xa6, 0xa3, 0xfd, 0xde, 0x74, 0xb4, 0x3f, 0x9b, 0x8e, 0x76, 0xf9, 0x3c, 0x4e, 0xc4, + 0xb7, 0xe5, 0x15, 0x9e, 0x72, 0xe6, 0x48, 0x1d, 0x47, 0xea, 0x38, 0xf9, 0xec, 0xc6, 0xb9, 0x1d, + 0xa8, 0x5f, 0xe0, 0xad, 0x72, 0xe2, 0x54, 0x1e, 0x2f, 0xfe, 0x06, 0x00, 0x00, 0xff, 0xff, 0x87, + 0xa1, 0xae, 0x17, 0x5c, 0x03, 0x00, 0x00, } func (m *MetricsReport) Marshal() (dAtA []byte, err error) { diff --git a/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto b/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto index e4cba46fd..fa3cf941d 100644 --- a/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto +++ b/test/performance/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto @@ -17,6 +17,10 @@ message MetricsReport { INSTANCE = 1; // Agent metric type AGENT = 2; + // Cache zone metric type + CACHE_ZONE = 3; + // Upstreams metric type + UPSTREAMS = 4; } // Provides meta information about the metrics Metadata meta = 1 [(gogoproto.jsontag) = "meta"]; diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/config/defaults.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/config/defaults.go index 5450c6e41..194a6deb0 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/config/defaults.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/config/defaults.go @@ -71,7 +71,7 @@ var ( BulkSize: 20, ReportInterval: 1 * time.Minute, CollectionInterval: 15 * time.Second, - Mode: "aggregation", + Mode: "aggregated", }, Features: agent_config.GetDefaultFeatures(), } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/aggregate.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/aggregate.go index 8bbff11c4..38a129749 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/aggregate.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/aggregate.go @@ -40,7 +40,7 @@ func dimChecksum(stats *proto.StatsEntity) string { // SaveCollections loops through one or more reports and get all the raw metrics for the Collections // Note this function operate on the Collections struct data directly. func SaveCollections(metricsCollections Collections, reports ...*proto.MetricsReport) Collections { - // could be multiple reports + // could be multiple reports, however they must all be of the same type. for _, report := range reports { metricsCollections.Count++ for _, stats := range report.GetData() { @@ -65,7 +65,7 @@ func SaveCollections(metricsCollections Collections, reports ...*proto.MetricsRe return metricsCollections } -func GenerateMetricsReport(metricsCollections Collections) *proto.MetricsReport { +func GenerateMetrics(metricsCollections Collections) []*proto.StatsEntity { results := make([]*proto.StatsEntity, 0, 200) @@ -77,11 +77,7 @@ func GenerateMetricsReport(metricsCollections Collections) *proto.MetricsReport )) } - return &proto.MetricsReport{ - Meta: &proto.Metadata{}, - Type: 0, - Data: results, - } + return results } func getAggregatedSimpleMetric(count int, internalMap map[string]float64) (simpleMetrics []*proto.SimpleMetric) { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/container.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/container.go index f01550a33..1beb937f7 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/container.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/container.go @@ -13,7 +13,6 @@ import ( log "github.com/sirupsen/logrus" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" @@ -27,7 +26,7 @@ var ( type ContainerCollector struct { sources []metrics.Source - buf chan *proto.StatsEntity + buf chan *metrics.StatsEntityWrapper dim *metrics.CommonDim env core.Environment } @@ -42,7 +41,7 @@ func NewContainerCollector(env core.Environment, conf *config.Config) *Container return &ContainerCollector{ sources: containerSources, - buf: make(chan *proto.StatsEntity, 65535), + buf: make(chan *metrics.StatsEntityWrapper, 65535), dim: metrics.NewCommonDim(env.NewHostInfo("agentVersion", &conf.Tags, conf.ConfigDirs, false), conf, ""), env: env, } @@ -59,7 +58,7 @@ func (c *ContainerCollector) collectMetrics(ctx context.Context) { wg.Wait() } -func (c *ContainerCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *ContainerCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.collectMetrics(ctx) @@ -69,7 +68,7 @@ func (c *ContainerCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m case <-ctx.Done(): return case sample := <-c.buf: - sample.Dimensions = append(commonDims, sample.Dimensions...) + sample.Data.Dimensions = append(commonDims, sample.Data.Dimensions...) select { case <-ctx.Done(): diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/nginx.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/nginx.go index 11f08ae39..93a3539ab 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/nginx.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/nginx.go @@ -11,7 +11,6 @@ import ( "context" "sync" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" @@ -25,7 +24,7 @@ var ( type NginxCollector struct { sources []metrics.NginxSource - buf chan *proto.StatsEntity + buf chan *metrics.StatsEntityWrapper dimensions *metrics.CommonDim collectorConf *metrics.NginxCollectorConfig env core.Environment @@ -40,7 +39,7 @@ func NewNginxCollector(conf *config.Config, env core.Environment, collectorConf return &NginxCollector{ sources: buildSources(dimensions, binary, collectorConf), - buf: make(chan *proto.StatsEntity, 65535), + buf: make(chan *metrics.StatsEntityWrapper, 65535), dimensions: dimensions, collectorConf: collectorConf, env: env, @@ -81,7 +80,7 @@ func (c *NginxCollector) collectMetrics(ctx context.Context) { wg.Wait() } -func (c *NginxCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.collectMetrics(ctx) for { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/system.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/system.go index b4bf7a2d8..3fcb054d4 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/system.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/collectors/system.go @@ -11,7 +11,6 @@ import ( "context" "sync" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" @@ -24,7 +23,7 @@ var ( type SystemCollector struct { sources []metrics.Source - buf chan *proto.StatsEntity + buf chan *metrics.StatsEntityWrapper dim *metrics.CommonDim env core.Environment } @@ -53,7 +52,7 @@ func NewSystemCollector(env core.Environment, conf *config.Config) *SystemCollec return &SystemCollector{ sources: systemSources, - buf: make(chan *proto.StatsEntity, 65535), + buf: make(chan *metrics.StatsEntityWrapper, 65535), dim: metrics.NewCommonDim(env.NewHostInfo("agentVersion", &conf.Tags, conf.ConfigDirs, false), conf, ""), env: env, } @@ -70,7 +69,7 @@ func (c *SystemCollector) collectMetrics(ctx context.Context) { wg.Wait() } -func (c *SystemCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *SystemCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.collectMetrics(ctx) @@ -80,7 +79,7 @@ func (c *SystemCollector) Collect(ctx context.Context, wg *sync.WaitGroup, m cha case <-ctx.Done(): return case sample := <-c.buf: - sample.Dimensions = append(commonDims, sample.Dimensions...) + sample.Data.Dimensions = append(commonDims, sample.Data.Dimensions...) select { case <-ctx.Done(): diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go index 1d5823b72..2cae8caf0 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/metrics_util.go @@ -15,23 +15,34 @@ import ( "time" "github.com/nginx/agent/sdk/v2/proto" + "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/gogo/protobuf/types" ) type Collector interface { - Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) + Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *StatsEntityWrapper) UpdateConfig(config *config.Config) } type Source interface { - Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) + Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *StatsEntityWrapper) } type NginxSource interface { - Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) + Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *StatsEntityWrapper) Update(dimensions *CommonDim, collectorConf *NginxCollectorConfig) Stop() } + +type StatsEntityWrapper struct { + Type proto.MetricsReport_Type + Data *proto.StatsEntity +} + +type MetricsReportBundle struct { + Data []*proto.MetricsReport +} + type NginxCollectorConfig struct { NginxId string StubStatus string @@ -44,6 +55,14 @@ type NginxCollectorConfig struct { ClientVersion int } +func NewStatsEntityWrapper(dims []*proto.Dimension, samples []*proto.SimpleMetric, seType proto.MetricsReport_Type) *StatsEntityWrapper { + return &StatsEntityWrapper{seType, &proto.StatsEntity{ + Timestamp: types.TimestampNow(), + Dimensions: dims, + Simplemetrics: samples, + }} +} + func NewStatsEntity(dims []*proto.Dimension, samples []*proto.SimpleMetric) *proto.StatsEntity { return &proto.StatsEntity{ Timestamp: types.TimestampNow(), @@ -366,3 +385,29 @@ func GetCalculationMap() map[string]string { "container.mem.oom.kill": "avg", } } + +func GenerateMetricsReportBundle(entities []*StatsEntityWrapper) core.Payload { + reportMap := make(map[proto.MetricsReport_Type]*proto.MetricsReport, 0) + + for _, entity := range entities { + if _, ok := reportMap[entity.Type]; !ok { + reportMap[entity.Type] = &proto.MetricsReport{ + Meta: &proto.Metadata{ + Timestamp: types.TimestampNow(), + }, + Type: entity.Type, + Data: make([]*proto.StatsEntity, 0), + } + } + reportMap[entity.Type].Data = append(reportMap[entity.Type].Data, entity.Data) + } + + if len(reportMap) > 0 { + bundle := &MetricsReportBundle{Data: []*proto.MetricsReport{}} + for _, report := range reportMap { + bundle.Data = append(bundle.Data, report) + } + return bundle + } + return nil +} diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/common.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/common.go index 8a1fa3fab..3041ee512 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/common.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/common.go @@ -88,11 +88,11 @@ func Delta(current, previous map[string]map[string]float64) map[string]map[strin return diff } -func SendNginxDownStatus(ctx context.Context, dims []*proto.Dimension, m chan<- *proto.StatsEntity) { +func SendNginxDownStatus(ctx context.Context, dims []*proto.Dimension, m chan<- *metrics.StatsEntityWrapper) { simpleMetrics := []*proto.SimpleMetric{newFloatMetric("nginx.status", float64(0))} select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity(dims, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE): } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/container_cpu.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/container_cpu.go index 6023d2aa3..474f6871c 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/container_cpu.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/container_cpu.go @@ -47,7 +47,7 @@ func NewContainerCPUSource(namespace string, basePath string) *ContainerCPU { return &ContainerCPU{basePath, cgroup.IsCgroupV2(basePath), NewMetricSourceLogger(), &namedMetric{namespace, CpuGroup}} } -func (c *ContainerCPU) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *ContainerCPU) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { log.Trace("Collecting container CPU metrics") defer wg.Done() @@ -205,7 +205,7 @@ func (c *ContainerCPU) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/container_mem.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/container_mem.go index 20b08f109..01d0bcd7c 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/container_mem.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/container_mem.go @@ -40,7 +40,7 @@ func NewContainerMemorySource(namespace string, basePath string) *ContainerMemor return &ContainerMemory{basePath, cgroup.IsCgroupV2(basePath), NewMetricSourceLogger(), &namedMetric{namespace, MemoryGroup}} } -func (c *ContainerMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *ContainerMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { log.Trace("Collecting container memory metrics") defer wg.Done() @@ -70,7 +70,7 @@ func (c *ContainerMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m cha select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/cpu.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/cpu.go index 4e1a8eaa2..a5053441d 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/cpu.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/cpu.go @@ -69,7 +69,7 @@ func diffTimeStat(t1, t2 cpu.TimesStat) cpu.TimesStat { } } -func (c *CPUTimes) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *CPUTimes) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() var simpleMetrics []*proto.SimpleMetric if c.isDocker { @@ -126,6 +126,6 @@ func (c *CPUTimes) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *pr select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go index 6de927d32..909ee4a85 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk.go @@ -30,7 +30,7 @@ func NewDiskSource(namespace string) *Disk { return &Disk{NewMetricSourceLogger(), &namedMetric{namespace, "disk"}, disks} } -func (c *Disk) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *Disk) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() for _, part := range c.disks { if part.Device == "" || part.Fstype == "" { @@ -54,7 +54,7 @@ func (c *Disk) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto. case <-ctx.Done(): return // mount point is not a common dim - case m <- metrics.NewStatsEntity([]*proto.Dimension{{Name: MOUNT_POINT, Value: part.Mountpoint}}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{{Name: MOUNT_POINT, Value: part.Mountpoint}}, simpleMetrics, proto.MetricsReport_SYSTEM): } } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk_io.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk_io.go index 30b13ea39..eaa961cd2 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk_io.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/disk_io.go @@ -37,7 +37,7 @@ func NewDiskIOSource(namespace string, env core.Environment) *DiskIO { return &DiskIO{namedMetric: &namedMetric{namespace, "io"}, env: env, diskIOStatsFunc: disk.IOCountersWithContext} } -func (dio *DiskIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (dio *DiskIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() dio.init.Do(func() { dio.diskDevs, _ = dio.env.DiskDevices() @@ -60,7 +60,7 @@ func (dio *DiskIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *pr // the Controller 3.x was labelling it wrong. However, changing this on the Analytics side // would involve a lot of changes (UI, API, Schema and Ingestion). So we are using mount_point // dimension for now. - case m <- metrics.NewStatsEntity([]*proto.Dimension{{Name: MOUNT_POINT, Value: k}}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{{Name: MOUNT_POINT, Value: k}}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/load.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/load.go index 29915ba4a..ca1e7ab7d 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/load.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/load.go @@ -27,7 +27,7 @@ func NewLoadSource(namespace string) *Load { return &Load{logger: NewMetricSourceLogger(), namedMetric: &namedMetric{namespace, "load"}, avgStatsFunc: load.Avg} } -func (c *Load) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *Load) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() loadStats, err := c.avgStatsFunc() if err != nil { @@ -43,6 +43,6 @@ func (c *Load) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto. select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/mem.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/mem.go index c02d441f9..c83e724ca 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/mem.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/mem.go @@ -38,7 +38,7 @@ func NewVirtualMemorySource(namespace string, env core.Environment) *VirtualMemo return &VirtualMemory{NewMetricSourceLogger(), &namedMetric{namespace, MemoryGroup}, statFunc} } -func (c *VirtualMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *VirtualMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() memstats, err := c.statFunc() if err != nil { @@ -66,6 +66,6 @@ func (c *VirtualMemory) Collect(ctx context.Context, wg *sync.WaitGroup, m chan< select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/net_io.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/net_io.go index 6e1b27b93..44bec29d8 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/net_io.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/net_io.go @@ -46,7 +46,7 @@ func NewNetIOSource(namespace string, env core.Environment) *NetIO { } } -func (nio *NetIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (nio *NetIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() nio.init.Do(func() { ifs, err := nio.newNetInterfaces(ctx) @@ -87,7 +87,7 @@ func (nio *NetIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *pro case <-ctx.Done(): return // network_interface is not a common dim - case m <- metrics.NewStatsEntity([]*proto.Dimension{{Name: NETWORK_INTERFACE, Value: k}}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{{Name: NETWORK_INTERFACE, Value: k}}, simpleMetrics, proto.MetricsReport_SYSTEM): } } @@ -106,7 +106,7 @@ func (nio *NetIO) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *pro nio.netOverflows = overflows simpleMetrics := nio.convertSamplesToSimpleMetrics(totalStats) - m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics) + m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM) nio.netIOStats = currentNetIOStats } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go index 4bcd71b79..fec3e7477 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_access_log.go @@ -67,7 +67,7 @@ type NginxAccessLog struct { binary core.NginxBinary nginxType string collectionInterval time.Duration - buf []*proto.StatsEntity + buf []*metrics.StatsEntityWrapper logger *MetricSourceLogger } @@ -88,7 +88,7 @@ func NewNginxAccessLog( binary, nginxType, collectionInterval, - []*proto.StatsEntity{}, + []*metrics.StatsEntityWrapper{}, NewMetricSourceLogger(), } @@ -105,7 +105,7 @@ func NewNginxAccessLog( return nginxAccessLog } -func (c *NginxAccessLog) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxAccessLog) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.collectLogStats(ctx, m) } @@ -149,7 +149,7 @@ func (c *NginxAccessLog) Stop() { log.Debugf("Stopping NginxAccessLog source for nginx id: %v", c.baseDimensions.NginxId) } -func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *proto.StatsEntity) { +func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *metrics.StatsEntityWrapper) { c.mu.Lock() defer c.mu.Unlock() logs := c.binary.GetAccessLogs() @@ -178,7 +178,7 @@ func (c *NginxAccessLog) collectLogStats(ctx context.Context, m chan<- *proto.St for _, stat := range c.buf { m <- stat } - c.buf = []*proto.StatsEntity{} + c.buf = []*metrics.StatsEntityWrapper{} } func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string) { @@ -337,7 +337,7 @@ func (c *NginxAccessLog) logStats(ctx context.Context, logFile, logFormat string httpCounters, upstreamCounters, upstreamCacheCounters = getDefaultCounters() gzipRatios, requestLengths, requestTimes, upstreamResponseLength, upstreamResponseTimes, upstreamConnectTimes, upstreamHeaderTimes = []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{}, []float64{} - c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics)) + c.buf = append(c.buf, metrics.NewStatsEntityWrapper(c.baseDimensions.ToDimensions(), simpleMetrics, proto.MetricsReport_INSTANCE)) mu.Unlock() diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_error_log.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_error_log.go index b9889ed59..1e8cd77ec 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_error_log.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_error_log.go @@ -9,11 +9,11 @@ package sources import ( "context" + "github.com/nginx/agent/sdk/v2/proto" re "regexp" "sync" "time" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/metrics" "github.com/nginx/agent/v2/src/core/metrics/sources/tailer" @@ -66,7 +66,7 @@ type NginxErrorLog struct { binary core.NginxBinary nginxType string collectionInterval time.Duration - buf []*proto.StatsEntity + buf []*metrics.StatsEntityWrapper } func NewNginxErrorLog( @@ -85,7 +85,7 @@ func NewNginxErrorLog( binary, nginxType, collectionInterval, - []*proto.StatsEntity{}, + []*metrics.StatsEntityWrapper{}, } logs := binary.GetErrorLogs() @@ -101,7 +101,7 @@ func NewNginxErrorLog( return nginxErrorLog } -func (c *NginxErrorLog) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxErrorLog) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.collectLogStats(ctx, m) @@ -145,7 +145,7 @@ func (c *NginxErrorLog) Update(dimensions *metrics.CommonDim, collectorConf *met } } -func (c *NginxErrorLog) collectLogStats(ctx context.Context, m chan<- *proto.StatsEntity) { +func (c *NginxErrorLog) collectLogStats(ctx context.Context, m chan<- *metrics.StatsEntityWrapper) { c.mu.Lock() defer c.mu.Unlock() logs := c.binary.GetErrorLogs() @@ -174,7 +174,7 @@ func (c *NginxErrorLog) collectLogStats(ctx context.Context, m chan<- *proto.Sta for _, stat := range c.buf { m <- stat } - c.buf = []*proto.StatsEntity{} + c.buf = []*metrics.StatsEntityWrapper{} } func (c *NginxErrorLog) logStats(ctx context.Context, logFile string) { @@ -231,7 +231,7 @@ func (c *NginxErrorLog) logStats(ctx context.Context, logFile string) { UpstreamResponseFailedMetricName: 0, } - c.buf = append(c.buf, metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics)) + c.buf = append(c.buf, metrics.NewStatsEntityWrapper(c.baseDimensions.ToDimensions(), simpleMetrics, proto.MetricsReport_INSTANCE)) mu.Unlock() diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_oss.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_oss.go index 51d021165..776e5288b 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_oss.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_oss.go @@ -34,7 +34,7 @@ func NewNginxOSS(baseDimensions *metrics.CommonDim, namespace, stubStatus string return &NginxOSS{baseDimensions: baseDimensions, stubStatus: stubStatus, namedMetric: &namedMetric{namespace: namespace}, logger: NewMetricSourceLogger()} } -func (c *NginxOSS) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxOSS) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.init.Do(func() { cl, err := client.NewNginxClient(&http.Client{}, c.stubStatus) @@ -108,7 +108,7 @@ func (c *NginxOSS) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *pr select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics): + case m <- metrics.NewStatsEntityWrapper(c.baseDimensions.ToDimensions(), simpleMetrics, proto.MetricsReport_INSTANCE): } c.prevStats = stats diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_plus.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_plus.go index 9a7b529ee..6b0e19b1c 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_plus.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_plus.go @@ -50,7 +50,7 @@ func NewNginxPlus(baseDimensions *metrics.CommonDim, nginxNamespace, plusNamespa return &NginxPlus{baseDimensions: baseDimensions, nginxNamespace: nginxNamespace, plusNamespace: plusNamespace, plusAPI: plusAPI, clientVersion: clientVersion, logger: NewMetricSourceLogger()} } -func (c *NginxPlus) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxPlus) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() c.init.Do(func() { cl, err := plusclient.NewNginxClientWithVersion(&http.Client{}, c.plusAPI, c.clientVersion) @@ -106,7 +106,7 @@ func (c *NginxPlus) Stop() { log.Debugf("Stopping NginxPlus source for nginx id: %v", c.baseDimensions.NginxId) } -func (c *NginxPlus) sendMetrics(ctx context.Context, m chan<- *proto.StatsEntity, entries ...*proto.StatsEntity) { +func (c *NginxPlus) sendMetrics(ctx context.Context, m chan<- *metrics.StatsEntityWrapper, entries ...*metrics.StatsEntityWrapper) { for _, entry := range entries { select { case <-ctx.Done(): @@ -116,7 +116,7 @@ func (c *NginxPlus) sendMetrics(ctx context.Context, m chan<- *proto.StatsEntity } } -func (c *NginxPlus) collectMetrics(stats, prevStats *plusclient.Stats) (entries []*proto.StatsEntity) { +func (c *NginxPlus) collectMetrics(stats, prevStats *plusclient.Stats) (entries []*metrics.StatsEntityWrapper) { entries = append(entries, c.instanceMetrics(stats, prevStats), c.commonMetrics(stats, prevStats), @@ -124,16 +124,17 @@ func (c *NginxPlus) collectMetrics(stats, prevStats *plusclient.Stats) (entries entries = append(entries, c.serverZoneMetrics(stats, prevStats)...) entries = append(entries, c.streamServerZoneMetrics(stats, prevStats)...) entries = append(entries, c.locationZoneMetrics(stats, prevStats)...) - entries = append(entries, c.cacheMetrics(stats, prevStats)...) - entries = append(entries, c.httpUpstreamMetrics(stats, prevStats)...) - entries = append(entries, c.streamUpstreamMetrics(stats, prevStats)...) entries = append(entries, c.slabMetrics(stats, prevStats)...) entries = append(entries, c.httpLimitConnsMetrics(stats, prevStats)...) entries = append(entries, c.httpLimitRequestMetrics(stats, prevStats)...) - return entries + entries = append(entries, c.cacheMetrics(stats, prevStats)...) + entries = append(entries, c.httpUpstreamMetrics(stats, prevStats)...) + entries = append(entries, c.streamUpstreamMetrics(stats, prevStats)...) + + return } -func (c *NginxPlus) instanceMetrics(stats, prevStats *plusclient.Stats) *proto.StatsEntity { +func (c *NginxPlus) instanceMetrics(stats, prevStats *plusclient.Stats) *metrics.StatsEntityWrapper { l := &namedMetric{namespace: c.nginxNamespace, group: ""} configGeneration := stats.NginxInfo.Generation - prevStats.NginxInfo.Generation @@ -147,11 +148,11 @@ func (c *NginxPlus) instanceMetrics(stats, prevStats *plusclient.Stats) *proto.S }) dims := c.baseDimensions.ToDimensions() - return metrics.NewStatsEntity(dims, simpleMetrics) + return metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE) } // commonMetrics uses the namespace of nginx because the metrics are common between oss and plus -func (c *NginxPlus) commonMetrics(stats, prevStats *plusclient.Stats) *proto.StatsEntity { +func (c *NginxPlus) commonMetrics(stats, prevStats *plusclient.Stats) *metrics.StatsEntityWrapper { l := &namedMetric{namespace: c.nginxNamespace, group: "http"} // For the case if nginx restarted (systemctl restart nginx), the stats counters were reset to zero, and @@ -182,10 +183,10 @@ func (c *NginxPlus) commonMetrics(stats, prevStats *plusclient.Stats) *proto.Sta }) dims := c.baseDimensions.ToDimensions() - return metrics.NewStatsEntity(dims, simpleMetrics) + return metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE) } -func (c *NginxPlus) sslMetrics(stats, prevStats *plusclient.Stats) *proto.StatsEntity { +func (c *NginxPlus) sslMetrics(stats, prevStats *plusclient.Stats) *metrics.StatsEntityWrapper { l := &namedMetric{namespace: c.plusNamespace, group: ""} sslHandshakes := stats.SSL.Handshakes - prevStats.SSL.Handshakes @@ -208,11 +209,11 @@ func (c *NginxPlus) sslMetrics(stats, prevStats *plusclient.Stats) *proto.StatsE }) dims := c.baseDimensions.ToDimensions() - return metrics.NewStatsEntity(dims, simpleMetrics) + return metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE) } -func (c *NginxPlus) serverZoneMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - zoneMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) serverZoneMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + zoneMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, sz := range stats.ServerZones { l := &namedMetric{namespace: c.plusNamespace, group: "http"} @@ -273,14 +274,14 @@ func (c *NginxPlus) serverZoneMetrics(stats, prevStats *plusclient.Stats) []*pro dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "server_zone", Value: name}) - zoneMetrics = append(zoneMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + zoneMetrics = append(zoneMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE)) } return zoneMetrics } -func (c *NginxPlus) streamServerZoneMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - zoneMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) streamServerZoneMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + zoneMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, ssz := range stats.StreamServerZones { l := &namedMetric{namespace: c.plusNamespace, group: "stream"} @@ -331,13 +332,13 @@ func (c *NginxPlus) streamServerZoneMetrics(stats, prevStats *plusclient.Stats) dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "server_zone", Value: name}) - zoneMetrics = append(zoneMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + zoneMetrics = append(zoneMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE)) } return zoneMetrics } -func (c *NginxPlus) locationZoneMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - zoneMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) locationZoneMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + zoneMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, lz := range stats.LocationZones { l := &namedMetric{namespace: c.plusNamespace, group: "http"} @@ -397,14 +398,14 @@ func (c *NginxPlus) locationZoneMetrics(stats, prevStats *plusclient.Stats) []*p dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "location_zone", Value: name}) - zoneMetrics = append(zoneMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + zoneMetrics = append(zoneMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE)) } return zoneMetrics } -func (c *NginxPlus) httpUpstreamMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - upstreamMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) httpUpstreamMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + upstreamMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, u := range stats.Upstreams { httpUpstreamHeaderTimes := []float64{} httpUpstreamResponseTimes := []float64{} @@ -496,7 +497,7 @@ func (c *NginxPlus) httpUpstreamMetrics(stats, prevStats *plusclient.Stats) []*p peerDims = append(peerDims, &proto.Dimension{Name: "upstream_zone", Value: u.Zone}) peerDims = append(peerDims, &proto.Dimension{Name: "peer.name", Value: peer.Name}) peerDims = append(peerDims, &proto.Dimension{Name: "peer.address", Value: peer.Server}) - upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntity(peerDims, simpleMetrics2)) + upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntityWrapper(peerDims, simpleMetrics2, proto.MetricsReport_UPSTREAMS)) } upstreamQueueOverflows := u.Queue.Overflows - prevStats.Upstreams[name].Queue.Overflows @@ -529,14 +530,14 @@ func (c *NginxPlus) httpUpstreamMetrics(stats, prevStats *plusclient.Stats) []*p upstreamDims := c.baseDimensions.ToDimensions() upstreamDims = append(upstreamDims, &proto.Dimension{Name: "upstream", Value: name}) upstreamDims = append(upstreamDims, &proto.Dimension{Name: "upstream_zone", Value: u.Zone}) - upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntity(upstreamDims, simpleMetrics)) + upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntityWrapper(upstreamDims, simpleMetrics, proto.MetricsReport_UPSTREAMS)) } return upstreamMetrics } -func (c *NginxPlus) streamUpstreamMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - upstreamMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) streamUpstreamMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + upstreamMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, u := range stats.StreamUpstreams { streamUpstreamResponseTimes := []float64{} streamUpstreamConnTimes := []float64{} @@ -605,7 +606,7 @@ func (c *NginxPlus) streamUpstreamMetrics(stats, prevStats *plusclient.Stats) [] peerDims = append(peerDims, &proto.Dimension{Name: "upstream_zone", Value: u.Zone}) peerDims = append(peerDims, &proto.Dimension{Name: "peer.name", Value: peer.Name}) peerDims = append(peerDims, &proto.Dimension{Name: "peer.address", Value: peer.Server}) - upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntity(peerDims, simpleMetrics2)) + upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntityWrapper(peerDims, simpleMetrics2, proto.MetricsReport_UPSTREAMS)) } simpleMetrics := l.convertSamplesToSimpleMetrics(map[string]float64{ @@ -629,14 +630,14 @@ func (c *NginxPlus) streamUpstreamMetrics(stats, prevStats *plusclient.Stats) [] upstreamDims := c.baseDimensions.ToDimensions() upstreamDims = append(upstreamDims, &proto.Dimension{Name: "upstream", Value: name}) upstreamDims = append(upstreamDims, &proto.Dimension{Name: "upstream_zone", Value: u.Zone}) - upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntity(upstreamDims, simpleMetrics)) + upstreamMetrics = append(upstreamMetrics, metrics.NewStatsEntityWrapper(upstreamDims, simpleMetrics, proto.MetricsReport_UPSTREAMS)) } return upstreamMetrics } -func (c *NginxPlus) cacheMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - zoneMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) cacheMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + zoneMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, ca := range stats.Caches { l := &namedMetric{namespace: c.plusNamespace, group: "cache"} @@ -718,15 +719,15 @@ func (c *NginxPlus) cacheMetrics(stats, prevStats *plusclient.Stats) []*proto.St dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "cache_zone", Value: name}) - zoneMetrics = append(zoneMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + zoneMetrics = append(zoneMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_CACHE_ZONE)) } return zoneMetrics } -func (c *NginxPlus) slabMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { +func (c *NginxPlus) slabMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { l := &namedMetric{namespace: c.plusNamespace, group: ""} - slabMetrics := make([]*proto.StatsEntity, 0) + slabMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, slab := range stats.Slabs { pages := slab.Pages @@ -746,7 +747,7 @@ func (c *NginxPlus) slabMetrics(stats, prevStats *plusclient.Stats) []*proto.Sta dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "zone", Value: name}) - slabMetrics = append(slabMetrics, metrics.NewStatsEntity(dims, slabSimpleMetrics)) + slabMetrics = append(slabMetrics, metrics.NewStatsEntityWrapper(dims, slabSimpleMetrics, proto.MetricsReport_INSTANCE)) for slotNum, slot := range slab.Slots { slotSimpleMetrics := l.convertSamplesToSimpleMetrics(map[string]float64{ @@ -755,15 +756,15 @@ func (c *NginxPlus) slabMetrics(stats, prevStats *plusclient.Stats) []*proto.Sta "slab.slots." + slotNum + ".reqs": float64(slot.Reqs), "slab.slots." + slotNum + ".used": float64(slot.Used), }) - slabMetrics = append(slabMetrics, metrics.NewStatsEntity(dims, slotSimpleMetrics)) + slabMetrics = append(slabMetrics, metrics.NewStatsEntityWrapper(dims, slotSimpleMetrics, proto.MetricsReport_INSTANCE)) } } return slabMetrics } -func (c *NginxPlus) httpLimitConnsMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - limitConnsMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) httpLimitConnsMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + limitConnsMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, lc := range stats.HTTPLimitConnections { l := &namedMetric{namespace: c.plusNamespace, group: "http"} @@ -774,14 +775,14 @@ func (c *NginxPlus) httpLimitConnsMetrics(stats, prevStats *plusclient.Stats) [] }) dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "limit_conn_zone", Value: name}) - limitConnsMetrics = append(limitConnsMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + limitConnsMetrics = append(limitConnsMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE)) } return limitConnsMetrics } -func (c *NginxPlus) httpLimitRequestMetrics(stats, prevStats *plusclient.Stats) []*proto.StatsEntity { - limitRequestMetrics := make([]*proto.StatsEntity, 0) +func (c *NginxPlus) httpLimitRequestMetrics(stats, prevStats *plusclient.Stats) []*metrics.StatsEntityWrapper { + limitRequestMetrics := make([]*metrics.StatsEntityWrapper, 0) for name, lr := range stats.HTTPLimitRequests { l := &namedMetric{namespace: c.plusNamespace, group: "http"} @@ -794,7 +795,7 @@ func (c *NginxPlus) httpLimitRequestMetrics(stats, prevStats *plusclient.Stats) }) dims := c.baseDimensions.ToDimensions() dims = append(dims, &proto.Dimension{Name: "limit_req_zone", Value: name}) - limitRequestMetrics = append(limitRequestMetrics, metrics.NewStatsEntity(dims, simpleMetrics)) + limitRequestMetrics = append(limitRequestMetrics, metrics.NewStatsEntityWrapper(dims, simpleMetrics, proto.MetricsReport_INSTANCE)) } return limitRequestMetrics diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_process.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_process.go index 35c55810f..4abc9f6c8 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_process.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_process.go @@ -43,7 +43,7 @@ func (c *NginxProcess) getNginxCount() float64 { return 0.0 } -func (c *NginxProcess) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxProcess) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() l := &namedMetric{namespace: PlusNamespace, group: ""} @@ -53,7 +53,7 @@ func (c *NginxProcess) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), countSimpleMetric): + case m <- metrics.NewStatsEntityWrapper(c.baseDimensions.ToDimensions(), countSimpleMetric, proto.MetricsReport_INSTANCE): } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_static.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_static.go index 86b0465ef..82cdde4f4 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_static.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_static.go @@ -11,7 +11,6 @@ import ( "context" "sync" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core/metrics" log "github.com/sirupsen/logrus" ) @@ -27,7 +26,7 @@ func NewNginxStatic(baseDimensions *metrics.CommonDim, namespace string) *NginxS return &NginxStatic{baseDimensions: baseDimensions, namedMetric: &namedMetric{namespace: namespace}} } -func (c *NginxStatic) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxStatic) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() SendNginxDownStatus(ctx, c.baseDimensions.ToDimensions(), m) diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_worker.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_worker.go index a64be2d23..ff619e2c4 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_worker.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/nginx_worker.go @@ -47,7 +47,7 @@ func NewNginxWorker(baseDimensions *metrics.CommonDim, } } -func (c *NginxWorker) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *NginxWorker) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { var err error defer wg.Done() childProcs := c.binary.GetChildProcesses() @@ -122,7 +122,7 @@ func (c *NginxWorker) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity(c.baseDimensions.ToDimensions(), simpleMetrics): + case m <- metrics.NewStatsEntityWrapper(c.baseDimensions.ToDimensions(), simpleMetrics, proto.MetricsReport_INSTANCE): } c.prevStats[pid] = stats diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/swap.go b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/swap.go index 638917201..027ef0c07 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/swap.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/core/metrics/sources/swap.go @@ -48,7 +48,7 @@ func NewSwapSource(namespace string, env core.Environment) *Swap { return &Swap{NewMetricSourceLogger(), &namedMetric{namespace, "swap"}, statFunc} } -func (c *Swap) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto.StatsEntity) { +func (c *Swap) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *metrics.StatsEntityWrapper) { defer wg.Done() swapStats, err := c.statFunc() if err != nil { @@ -73,6 +73,6 @@ func (c *Swap) Collect(ctx context.Context, wg *sync.WaitGroup, m chan<- *proto. select { case <-ctx.Done(): - case m <- metrics.NewStatsEntity([]*proto.Dimension{}, simpleMetrics): + case m <- metrics.NewStatsEntityWrapper([]*proto.Dimension{}, simpleMetrics, proto.MetricsReport_SYSTEM): } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/prometheus-metrics/prometheus_exporter.go b/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/prometheus-metrics/prometheus_exporter.go index 7215a7dbf..313611bbc 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/prometheus-metrics/prometheus_exporter.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/extensions/prometheus-metrics/prometheus_exporter.go @@ -10,19 +10,22 @@ import ( ) type Exporter struct { - latestMetricReport *proto.MetricsReport + latestMetricReports *metrics.MetricsReportBundle } func NewExporter(report *proto.MetricsReport) *Exporter { - return &Exporter{latestMetricReport: report} + return &Exporter{latestMetricReports: &metrics.MetricsReportBundle{Data: []*proto.MetricsReport{report}}} } -func (e *Exporter) SetLatestMetricReport(latest *proto.MetricsReport) { - e.latestMetricReport = latest +func (e *Exporter) SetLatestMetricReport(latest *metrics.MetricsReportBundle) { + e.latestMetricReports = latest } -func (e *Exporter) GetLatestMetricReport() *proto.MetricsReport { - return e.latestMetricReport +func (e *Exporter) GetLatestMetricReports() (reports []*proto.MetricsReport) { + for _, report := range e.latestMetricReports.Data { + reports = append(reports, report) + } + return } func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { @@ -40,9 +43,11 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { } func (e *Exporter) Collect(ch chan<- prometheus.Metric) { - for _, statsEntity := range e.latestMetricReport.Data { - for _, metric := range statsEntity.Simplemetrics { - ch <- createPrometheusMetric(metric, statsEntity.GetDimensions()) + for _, report := range e.latestMetricReports.Data { + for _, statsEntity := range report.Data { + for _, metric := range statsEntity.Simplemetrics { + ch <- createPrometheusMetric(metric, statsEntity.GetDimensions()) + } } } } diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/agent_api.go b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/agent_api.go index a3a626ddd..26f25bbf5 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/agent_api.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/agent_api.go @@ -14,6 +14,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/nginx/agent/v2/src/core/metrics" "io" "net/http" "os" @@ -165,10 +166,10 @@ func (a *AgentAPI) Process(message *core.Message) { } case core.MetricReport: switch response := message.Data().(type) { - case *proto.MetricsReport: + case *metrics.MetricsReportBundle: a.exporter.SetLatestMetricReport(response) default: - log.Warnf("Unknown MetricReport type: %T(%v)", message.Data(), message.Data()) + log.Warnf("Unknown MetricReportBundle type: %T(%v)", message.Data(), message.Data()) } case core.NginxConfigValidationPending, core.NginxConfigApplyFailed, core.NginxConfigApplySucceeded: switch response := message.Data().(type) { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics.go b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics.go index 118d3aa3e..8895f37cf 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics.go @@ -12,12 +12,10 @@ import ( "sync" "time" - "github.com/gogo/protobuf/types" log "github.com/sirupsen/logrus" "go.uber.org/atomic" "github.com/nginx/agent/sdk/v2" - "github.com/nginx/agent/sdk/v2/proto" "github.com/nginx/agent/v2/src/core" "github.com/nginx/agent/v2/src/core/config" "github.com/nginx/agent/v2/src/core/metrics" @@ -32,7 +30,7 @@ type Metrics struct { ticker *time.Ticker interval time.Duration collectors []metrics.Collector - buf chan *proto.StatsEntity + buf chan *metrics.StatsEntityWrapper errors chan error collectorConfigsMap map[string]*metrics.NginxCollectorConfig ctx context.Context @@ -53,7 +51,7 @@ func NewMetrics(config *config.Config, env core.Environment, binary core.NginxBi collectorsUpdate: atomic.NewBool(false), ticker: time.NewTicker(config.AgentMetrics.CollectionInterval), interval: config.AgentMetrics.CollectionInterval, - buf: make(chan *proto.StatsEntity, 4096), + buf: make(chan *metrics.StatsEntityWrapper, 4096), errors: make(chan error), collectorConfigsMap: collectorConfigsMap, wg: sync.WaitGroup{}, @@ -184,26 +182,22 @@ func (m *Metrics) metricsGoroutine() { } return case <-m.ticker.C: - report := &proto.MetricsReport{ - Meta: &proto.Metadata{ - Timestamp: types.TimestampNow(), - }, - Type: proto.MetricsReport_SYSTEM, - Data: m.collectStats(), + stats := m.collectStats() + if bundle := metrics.GenerateMetricsReportBundle(stats); bundle != nil { + m.pipeline.Process(core.NewMessage(core.MetricReport, bundle)) } - - m.pipeline.Process(core.NewMessage(core.MetricReport, report)) if m.collectorsUpdate.Load() { m.ticker = time.NewTicker(m.conf.AgentMetrics.CollectionInterval) m.collectorsUpdate.Store(false) } + case err := <-m.errors: log.Errorf("Error in metricsGoroutine %v", err) } } } -func (m *Metrics) collectStats() (stats []*proto.StatsEntity) { +func (m *Metrics) collectStats() (stats []*metrics.StatsEntityWrapper) { // setups a collect duration of half-time of the poll interval ctx, cancel := context.WithTimeout(m.ctx, m.interval/2) defer cancel() @@ -220,6 +214,7 @@ func (m *Metrics) collectStats() (stats []*proto.StatsEntity) { } // wait until all the collection go routines are done, which either context timeout or exit wg.Wait() + for len(m.buf) > 0 { // drain the buf, since our sources/collectors are all done, we can rely on buffer length select { @@ -233,8 +228,9 @@ func (m *Metrics) collectStats() (stats []*proto.StatsEntity) { stats = append(stats, stat) } } + log.Debugf("collected %d entries in %s (ctx error=%t)", len(stats), time.Since(start), ctx.Err() != nil) - return stats + return } func (m *Metrics) registerStatsSources() { diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics_sender.go b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics_sender.go index 0b47761c3..d982ee26c 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics_sender.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics_sender.go @@ -79,9 +79,7 @@ func (r *MetricsSender) Process(msg *core.Message) { err := r.reporter.Send(r.ctx, message) if err != nil { - log.Errorf("Failed to send MetricsReport: %v, data: %+v", err, report) - } else { - log.Tracef("MetricsReport sent, %v", report) + log.Errorf("Failed to send MetricsReport: %v", err) } case *models.EventReport: err := r.reporter.Send(r.ctx, client.MessageFromEvents(report)) diff --git a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics_throlling.go b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics_throlling.go index 373e8950c..1b046762c 100644 --- a/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics_throlling.go +++ b/test/performance/vendor/github.com/nginx/agent/v2/src/plugins/metrics_throlling.go @@ -9,6 +9,7 @@ package plugins import ( "context" + "github.com/gogo/protobuf/types" "sync" "time" @@ -33,7 +34,7 @@ type MetricsThrottle struct { reportsReady *atomic.Bool collectorsUpdate *atomic.Bool metricsAggregation bool - metricsCollections metrics.Collections + metricsCollections map[proto.MetricsReport_Type]*metrics.Collections ctx context.Context wg sync.WaitGroup mu sync.Mutex @@ -43,10 +44,6 @@ type MetricsThrottle struct { } func NewMetricsThrottle(conf *config.Config, env core.Environment) *MetricsThrottle { - metricsCollections := metrics.Collections{ - Count: 0, - Data: make(map[string]metrics.PerDimension), - } return &MetricsThrottle{ metricBuffer: make([]core.Payload, 0), @@ -55,7 +52,7 @@ func NewMetricsThrottle(conf *config.Config, env core.Environment) *MetricsThrot reportsReady: atomic.NewBool(false), collectorsUpdate: atomic.NewBool(false), metricsAggregation: conf.AgentMetrics.Mode == "aggregated", - metricsCollections: metricsCollections, + metricsCollections: make(map[proto.MetricsReport_Type]*metrics.Collections, 0), wg: sync.WaitGroup{}, env: env, conf: conf, @@ -92,21 +89,42 @@ func (r *MetricsThrottle) Process(msg *core.Message) { return case msg.Exact(core.MetricReport): if r.metricsAggregation { - switch report := msg.Data().(type) { - case *proto.MetricsReport: - r.mu.Lock() - r.metricsCollections = metrics.SaveCollections(r.metricsCollections, report) - r.mu.Unlock() - log.Debug("MetricsThrottle: Metrics collection saved") - r.reportsReady.Store(true) + switch bundle := msg.Data().(type) { + case *metrics.MetricsReportBundle: + if len(bundle.Data) > 0 { + r.mu.Lock() + for _, report := range bundle.Data { + if len(report.Data) > 0 { + if _, ok := r.metricsCollections[report.Type]; !ok { + r.metricsCollections[report.Type] = &metrics.Collections{ + Count: 0, + Data: make(map[string]metrics.PerDimension), + } + } + collection := metrics.SaveCollections(*r.metricsCollections[report.Type], report) + r.metricsCollections[report.Type] = &collection + log.Debugf("MetricsThrottle: Metrics collection saved [Type: %d]", report.Type) + } + } + r.mu.Unlock() + r.reportsReady.Store(true) + } } } else { - r.metricBuffer = append(r.metricBuffer, msg.Data()) + switch bundle := msg.Data().(type) { + case *metrics.MetricsReportBundle: + if len(bundle.Data) > 0 { + for _, report := range bundle.Data { + if len(report.Data) > 0 { + r.metricBuffer = append(r.metricBuffer, report) + } + } + } + } + log.Tracef("MetricsThrottle buffer size: %d of %d", len(r.metricBuffer), r.BulkSize) if len(r.metricBuffer) >= r.BulkSize { log.Info("MetricsThrottle buffer flush") - r.messagePipeline.Process( - core.NewMessage(core.CommMetrics, r.metricBuffer), - ) + r.messagePipeline.Process(core.NewMessage(core.CommMetrics, r.metricBuffer)) r.metricBuffer = make([]core.Payload, 0) } } @@ -134,10 +152,10 @@ func (r *MetricsThrottle) metricsReportGoroutine(ctx context.Context, wg *sync.W } return case <-r.ticker.C: - aggregatedReport := r.getAggregatedReport() - r.messagePipeline.Process( - core.NewMessage(core.CommMetrics, []core.Payload{aggregatedReport}), - ) + reports := r.getAggregatedReports() + if len(reports) > 0 { + r.messagePipeline.Process(core.NewMessage(core.CommMetrics, reports)) + } if r.collectorsUpdate.Load() { r.BulkSize = r.conf.AgentMetrics.BulkSize r.metricsAggregation = r.conf.AgentMetrics.Mode == "aggregated" @@ -167,13 +185,23 @@ func (r *MetricsThrottle) syncAgentConfigChange() { r.conf = conf } -func (r *MetricsThrottle) getAggregatedReport() *proto.MetricsReport { +func (r *MetricsThrottle) getAggregatedReports() (reports []core.Payload) { r.mu.Lock() defer r.mu.Unlock() - report := metrics.GenerateMetricsReport(r.metricsCollections) - r.metricsCollections = metrics.Collections{ - Count: 0, - Data: make(map[string]metrics.PerDimension), + + for reportType, collection := range r.metricsCollections { + reports = append(reports, &proto.MetricsReport{ + Meta: &proto.Metadata{ + Timestamp: types.TimestampNow(), + }, + Type: reportType, + Data: metrics.GenerateMetrics(*collection), + }) + r.metricsCollections[reportType] = &metrics.Collections{ + Count: 0, + Data: make(map[string]metrics.PerDimension), + } } - return report + + return } diff --git a/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go b/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go index 06fe5c328..46e46326b 100644 --- a/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go +++ b/vendor/github.com/nginx/agent/sdk/v2/client/metric_reporter.go @@ -157,7 +157,7 @@ func (r *metricReporter) Send(ctx context.Context, message Message) error { return r.handleGrpcError("Metric Reporter Channel Send", err) } - log.Tracef("MetricReporter sent metrics report %v", report) + log.Tracef("MetricReporter sent metrics report [Type: %d] %+v", report.Type, report) return nil }) diff --git a/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go b/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go index ee8c82794..e6ad339ef 100644 --- a/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go +++ b/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.pb.go @@ -35,18 +35,26 @@ const ( MetricsReport_INSTANCE MetricsReport_Type = 1 // Agent metric type MetricsReport_AGENT MetricsReport_Type = 2 + // Cache zone metric type + MetricsReport_CACHE_ZONE MetricsReport_Type = 3 + // Upstreams metric type + MetricsReport_UPSTREAMS MetricsReport_Type = 4 ) var MetricsReport_Type_name = map[int32]string{ 0: "SYSTEM", 1: "INSTANCE", 2: "AGENT", + 3: "CACHE_ZONE", + 4: "UPSTREAMS", } var MetricsReport_Type_value = map[string]int32{ - "SYSTEM": 0, - "INSTANCE": 1, - "AGENT": 2, + "SYSTEM": 0, + "INSTANCE": 1, + "AGENT": 2, + "CACHE_ZONE": 3, + "UPSTREAMS": 4, } func (x MetricsReport_Type) String() string { @@ -318,36 +326,38 @@ func init() { func init() { proto.RegisterFile("metrics.proto", fileDescriptor_6039342a2ba47b72) } var fileDescriptor_6039342a2ba47b72 = []byte{ - // 460 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x52, 0x41, 0x6f, 0xd3, 0x30, - 0x14, 0xc6, 0x6d, 0x36, 0x9a, 0xd7, 0x76, 0x2a, 0x3e, 0x55, 0xd5, 0x68, 0xaa, 0x1e, 0x50, 0x11, - 0x92, 0x23, 0x15, 0x21, 0x10, 0x88, 0xc3, 0xc2, 0x2a, 0xc4, 0xa4, 0xf4, 0xe0, 0xe4, 0x32, 0x6e, - 0x5e, 0xeb, 0x85, 0x68, 0x75, 0x1c, 0xd5, 0xee, 0x44, 0x6f, 0xfc, 0x3c, 0x8e, 0xfc, 0x82, 0x0a, - 0xf5, 0xd8, 0xff, 0x80, 0x84, 0x62, 0xa7, 0x5b, 0x27, 0xc2, 0x61, 0x97, 0x67, 0x3f, 0xeb, 0x7d, - 0xdf, 0x7b, 0xef, 0xfb, 0x0c, 0x6d, 0xc1, 0xf5, 0x32, 0x9d, 0x29, 0x92, 0x2f, 0xa5, 0x96, 0x18, - 0x5f, 0xbf, 0x21, 0x59, 0x92, 0x66, 0xdf, 0x09, 0x4b, 0x78, 0xa6, 0x89, 0x9a, 0xdf, 0xf4, 0x5a, - 0x33, 0x29, 0x84, 0xcc, 0x6c, 0x45, 0x0f, 0x12, 0x99, 0xc8, 0xf2, 0xee, 0x25, 0x52, 0x26, 0x0b, - 0xee, 0x9b, 0xec, 0x6a, 0x75, 0xed, 0xeb, 0x54, 0x70, 0xa5, 0x99, 0xc8, 0x6d, 0xc1, 0xf0, 0x0f, - 0x82, 0x76, 0x68, 0x1b, 0x50, 0x9e, 0xcb, 0xa5, 0xc6, 0xef, 0xc1, 0x11, 0x5c, 0xb3, 0x2e, 0x1a, - 0xa0, 0x51, 0x73, 0x7c, 0x4a, 0xfe, 0xed, 0x47, 0x42, 0xae, 0xd9, 0x9c, 0x69, 0x16, 0x34, 0x76, - 0x1b, 0xcf, 0x54, 0x53, 0x13, 0xf1, 0x39, 0x38, 0x7a, 0x9d, 0xf3, 0x6e, 0x6d, 0x80, 0x46, 0x27, - 0xe3, 0x17, 0xff, 0xc1, 0xde, 0x37, 0x23, 0xf1, 0x3a, 0xe7, 0x96, 0xa5, 0xc0, 0x51, 0x13, 0xf1, - 0x47, 0x70, 0x0a, 0xf6, 0x6e, 0x7d, 0x50, 0x1f, 0x35, 0xc7, 0x5e, 0x15, 0x4b, 0xa4, 0x99, 0x56, - 0x93, 0x4c, 0xa7, 0x7a, 0x6d, 0xe1, 0x05, 0x80, 0x9a, 0x38, 0x7c, 0x05, 0x4e, 0x41, 0x8b, 0x01, - 0x8e, 0xa3, 0xcb, 0x28, 0x9e, 0x84, 0x9d, 0x27, 0xb8, 0x05, 0x8d, 0x2f, 0xd3, 0x28, 0x3e, 0x9b, - 0x7e, 0x9a, 0x74, 0x10, 0x76, 0xe1, 0xe8, 0xec, 0xf3, 0x64, 0x1a, 0x77, 0x6a, 0xc3, 0x10, 0x5a, - 0x51, 0x2a, 0xf2, 0x05, 0xb7, 0x73, 0xe1, 0x53, 0x70, 0x32, 0x26, 0xb8, 0xd9, 0xde, 0xb5, 0xd4, - 0x45, 0x4e, 0x4d, 0xc4, 0x1e, 0x1c, 0xdd, 0xb2, 0xc5, 0xca, 0x2e, 0x88, 0x02, 0x77, 0xb7, 0xf1, - 0xec, 0x03, 0xb5, 0xc7, 0xf0, 0x02, 0xdc, 0xf3, 0x54, 0xf0, 0x4c, 0xa5, 0x32, 0x7b, 0x0c, 0x97, - 0x5b, 0xc1, 0xf5, 0xa3, 0x06, 0xcd, 0x83, 0x3d, 0xf1, 0x3b, 0x70, 0xef, 0xdc, 0x2b, 0xdd, 0xe9, - 0x11, 0xeb, 0x2f, 0xd9, 0xfb, 0x4b, 0xe2, 0x7d, 0x05, 0xbd, 0x2f, 0xc6, 0x21, 0xc0, 0x7c, 0x3f, - 0x95, 0x2a, 0x65, 0x7d, 0x5e, 0x25, 0xeb, 0xdd, 0xec, 0xc1, 0xc9, 0x6e, 0xe3, 0x1d, 0x80, 0xe8, - 0xc1, 0x1d, 0x5f, 0x42, 0x5b, 0x19, 0xcd, 0xca, 0x9f, 0xd9, 0x75, 0x0c, 0xe3, 0xa0, 0xd2, 0xa8, - 0x03, 0x71, 0x83, 0x67, 0xbb, 0x8d, 0xf7, 0x10, 0x4a, 0x1f, 0xa6, 0x17, 0x4e, 0xa3, 0xd6, 0xa9, - 0xd3, 0xa7, 0x65, 0x1a, 0xbc, 0xfd, 0xb9, 0xed, 0xa3, 0x5f, 0xdb, 0x3e, 0xfa, 0xbd, 0xed, 0xa3, - 0xaf, 0x2f, 0x93, 0x54, 0x7f, 0x5b, 0x5d, 0x91, 0x99, 0x14, 0xbe, 0xe9, 0xe3, 0x9b, 0x3e, 0xbe, - 0x9a, 0xdf, 0xf8, 0xb7, 0x63, 0xfb, 0xc7, 0x3f, 0x58, 0x25, 0x8e, 0xcd, 0xf1, 0xfa, 0x6f, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xe6, 0xcd, 0x1e, 0xb5, 0x3d, 0x03, 0x00, 0x00, + // 487 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xc5, 0x8e, 0x5b, 0xe2, 0xc9, 0x87, 0xcc, 0x9e, 0xa2, 0xa8, 0xc4, 0x51, 0x0e, 0x28, 0x5c, + 0xd6, 0x52, 0x10, 0x02, 0x81, 0x38, 0xc4, 0xa9, 0x05, 0x44, 0x72, 0x40, 0x6b, 0x73, 0x68, 0x2f, + 0xc8, 0x4d, 0xb6, 0xc6, 0x6a, 0xd6, 0x6b, 0xc5, 0x9b, 0x8a, 0xdc, 0xf8, 0x21, 0xfc, 0x20, 0x8e, + 0xfc, 0x82, 0x08, 0xe5, 0x98, 0x5f, 0x81, 0xbc, 0xeb, 0xb4, 0x89, 0x08, 0x87, 0x5e, 0x66, 0x77, + 0xac, 0x79, 0xef, 0x8d, 0xdf, 0xb3, 0xa1, 0xc1, 0xa8, 0x58, 0x24, 0xd3, 0x1c, 0x67, 0x0b, 0x2e, + 0x38, 0x42, 0xd7, 0x2f, 0x71, 0x1a, 0x27, 0xe9, 0x77, 0x1c, 0xc5, 0x34, 0x15, 0x38, 0x9f, 0xdd, + 0xb4, 0xeb, 0x53, 0xce, 0x18, 0x4f, 0xd5, 0x44, 0x1b, 0x62, 0x1e, 0xf3, 0xf2, 0x6e, 0xc7, 0x9c, + 0xc7, 0x73, 0xea, 0xc8, 0xee, 0x6a, 0x79, 0xed, 0x88, 0x84, 0xd1, 0x5c, 0x44, 0x2c, 0x53, 0x03, + 0xbd, 0x9f, 0x3a, 0x34, 0x7c, 0x25, 0x40, 0x68, 0xc6, 0x17, 0x02, 0xbd, 0x01, 0x83, 0x51, 0x11, + 0xb5, 0xb4, 0xae, 0xd6, 0xaf, 0x0d, 0xce, 0xf0, 0xbf, 0x7a, 0xd8, 0xa7, 0x22, 0x9a, 0x45, 0x22, + 0x72, 0xab, 0xdb, 0xb5, 0x2d, 0xa7, 0x89, 0xac, 0xe8, 0x1c, 0x0c, 0xb1, 0xca, 0x68, 0x4b, 0xef, + 0x6a, 0xfd, 0xe6, 0xe0, 0xd9, 0x7f, 0xb0, 0xf7, 0x62, 0x38, 0x5c, 0x65, 0x54, 0xb1, 0x14, 0x38, + 0x22, 0x2b, 0x7a, 0x07, 0x46, 0xc1, 0xde, 0xaa, 0x74, 0x2b, 0xfd, 0xda, 0xc0, 0x3e, 0xc6, 0x12, + 0x88, 0x48, 0xe4, 0x5e, 0x2a, 0x12, 0xb1, 0x52, 0xf0, 0x02, 0x40, 0x64, 0xed, 0x8d, 0xc1, 0x28, + 0x68, 0x11, 0xc0, 0x69, 0x70, 0x11, 0x84, 0x9e, 0x6f, 0x3d, 0x42, 0x75, 0xa8, 0x7e, 0x9c, 0x04, + 0xe1, 0x70, 0x32, 0xf2, 0x2c, 0x0d, 0x99, 0x70, 0x32, 0x7c, 0xef, 0x4d, 0x42, 0x4b, 0x47, 0x4d, + 0x80, 0xd1, 0x70, 0xf4, 0xc1, 0xfb, 0x7a, 0xf9, 0x69, 0xe2, 0x59, 0x15, 0xd4, 0x00, 0xf3, 0xcb, + 0xe7, 0x20, 0x24, 0xde, 0xd0, 0x0f, 0x2c, 0xa3, 0xe7, 0x43, 0x3d, 0x48, 0x58, 0x36, 0xa7, 0x6a, + 0x6d, 0x74, 0x06, 0x46, 0x1a, 0x31, 0x2a, 0xcd, 0x31, 0x95, 0x72, 0xd1, 0x13, 0x59, 0x91, 0x0d, + 0x27, 0xb7, 0xd1, 0x7c, 0xa9, 0xde, 0x5f, 0x73, 0xcd, 0xed, 0xda, 0x56, 0x0f, 0x88, 0x3a, 0x7a, + 0x63, 0x30, 0xcf, 0x13, 0x46, 0xd3, 0x3c, 0xe1, 0xe9, 0x43, 0xb8, 0xcc, 0x23, 0x5c, 0x3f, 0x74, + 0xa8, 0xed, 0xd9, 0x80, 0x5e, 0x83, 0x79, 0x17, 0x6e, 0x19, 0x5e, 0x1b, 0xab, 0xf8, 0xf1, 0x2e, + 0x7e, 0x1c, 0xee, 0x26, 0xc8, 0xfd, 0x30, 0xf2, 0x01, 0x66, 0xbb, 0xad, 0xf2, 0xd2, 0xf5, 0xa7, + 0xc7, 0x5c, 0xbf, 0xdb, 0xdd, 0x6d, 0x6e, 0xd7, 0xf6, 0x1e, 0x88, 0xec, 0xdd, 0xd1, 0x05, 0x34, + 0x72, 0xe9, 0x59, 0xf9, 0xe1, 0xb6, 0x0c, 0xc9, 0xd8, 0x3d, 0x9a, 0xe3, 0x9e, 0xb9, 0xee, 0x93, + 0xed, 0xda, 0x3e, 0x84, 0x92, 0xc3, 0x76, 0x6c, 0x54, 0x75, 0xab, 0x42, 0x1e, 0x97, 0xad, 0xfb, + 0xea, 0xd7, 0xa6, 0xa3, 0xfd, 0xde, 0x74, 0xb4, 0x3f, 0x9b, 0x8e, 0x76, 0xf9, 0x3c, 0x4e, 0xc4, + 0xb7, 0xe5, 0x15, 0x9e, 0x72, 0xe6, 0x48, 0x1d, 0x47, 0xea, 0x38, 0xf9, 0xec, 0xc6, 0xb9, 0x1d, + 0xa8, 0x5f, 0xe0, 0xad, 0x72, 0xe2, 0x54, 0x1e, 0x2f, 0xfe, 0x06, 0x00, 0x00, 0xff, 0xff, 0x87, + 0xa1, 0xae, 0x17, 0x5c, 0x03, 0x00, 0x00, } func (m *MetricsReport) Marshal() (dAtA []byte, err error) { diff --git a/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto b/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto index e4cba46fd..fa3cf941d 100644 --- a/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto +++ b/vendor/github.com/nginx/agent/sdk/v2/proto/metrics.proto @@ -17,6 +17,10 @@ message MetricsReport { INSTANCE = 1; // Agent metric type AGENT = 2; + // Cache zone metric type + CACHE_ZONE = 3; + // Upstreams metric type + UPSTREAMS = 4; } // Provides meta information about the metrics Metadata meta = 1 [(gogoproto.jsontag) = "meta"];