From 1dce06a1129792bc3ca3d697e5ef2d9d5f1fb203 Mon Sep 17 00:00:00 2001 From: filippomachi Date: Fri, 4 Jul 2025 21:22:51 +0200 Subject: [PATCH 1/3] [MetricBeat][StackMonitoring] fetching index summary from node/_stats endpoint (#45049) Fetching index summary metricset from `_nodes/stats` endpoint instead of `_stats` so that as indexes are deleted, the summary stats are not effectively lost during the next poll because nodes cache their data. Gaps can still appear in charts if nodes are restarted (as their cache is zeroed out). (cherry picked from commit 63a537a17839ef23b0cd4cd7d62e708319374b61) # Conflicts: # metricbeat/module/elasticsearch/index_summary/index_summary.go --- CHANGELOG.next.asciidoc | 2 + .../_meta/test/expected_event_7.17.json | 97 +++++++ .../_meta/test/expected_event_8.17.json | 97 +++++++ .../_meta/test/expected_event_xpack_7.17.json | 97 +++++++ .../_meta/test/expected_event_xpack_8.17.json | 97 +++++++ .../_meta/test/node_stats_empty.json | 4 + .../_meta/test/node_stats_v717.json | 76 +++++ .../test/node_stats_v717_field_as_string.json | 76 +++++ .../_meta/test/node_stats_v817.json | 164 +++++++++++ .../test/node_stats_v817_missing_block.json | 160 +++++++++++ .../test/node_stats_v817_missing_fields.json | 163 +++++++++++ .../elasticsearch/index_summary/data.go | 264 +++++++++++++----- .../elasticsearch/index_summary/data_test.go | 125 ++++++++- .../index_summary/index_summary.go | 28 +- .../index_summary/index_summary_test.go | 35 +++ metricbeat/module/elasticsearch/testing.go | 83 +++++- 16 files changed, 1468 insertions(+), 100 deletions(-) create mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_7.17.json create mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_8.17.json create mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_7.17.json create mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_8.17.json create mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_empty.json create mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717.json create mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717_field_as_string.json create mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817.json create mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_block.json create mode 100644 metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_fields.json create mode 100644 metricbeat/module/elasticsearch/index_summary/index_summary_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0b64fb128ba8..1f9a192fd6fd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -34,6 +34,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Metricbeat* +- Change index summary metricset to use `_nodes/stats` API instead of `_stats` API to avoid data gaps. {pull}45049[45049] +- Add support for `_nodes/stats` URIs that work with legacy versions of Elasticsearch {pull}44307[44307] - Setting period for counter cache for Prometheus remote_write at least to 60sec {pull}38553[38553] - Remove fallback to the node limit for the `kubernetes.pod.cpu.usage.limit.pct` and `kubernetes.pod.memory.usage.limit.pct` metrics calculation - Add support for Kibana status metricset in v8 format {pull}40275[40275] diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_7.17.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_7.17.json new file mode 100644 index 000000000000..e622a4f26b2d --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_7.17.json @@ -0,0 +1,97 @@ +{ + "RootFields": { + "service": { + "name": "elasticsearch" + } + }, + "ModuleFields": { + "cluster": { + "id": "1234", + "name": "helloworld" + } + }, + "MetricSetFields": { + "primaries": { + "docs": { + "count": 464, + "deleted": 252 + }, + "store": { + "size": { + "bytes": 85621934 + }, + "total_data_set_size": { + "bytes": 85621934 + } + }, + "indexing": { + "index": { + "count": 390, + "time": { + "ms": 1815 + } + } + }, + "search": { + "query": { + "count": 313, + "time": { + "ms": 806 + } + } + }, + "segments": { + "count": 80, + "memory": { + "bytes": 114024 + } + } + }, + "total": { + "docs": { + "count": 464, + "deleted": 252 + }, + "store": { + "size": { + "bytes": 85621934 + }, + "total_data_set_size": { + "bytes": 85621934 + } + }, + "indexing": { + "index": { + "count": 390, + "time": { + "ms": 1815 + } + } + }, + "search": { + "query": { + "count": 313, + "time": { + "ms": 806 + } + } + }, + "segments": { + "count": 80, + "memory": { + "bytes": 114024 + } + } + } + }, + "Index": ".monitoring-es-8-mb", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_8.17.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_8.17.json new file mode 100644 index 000000000000..82d3cf3778d0 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_8.17.json @@ -0,0 +1,97 @@ +{ + "RootFields": { + "service": { + "name": "elasticsearch" + } + }, + "ModuleFields": { + "cluster": { + "id": "1234", + "name": "helloworld" + } + }, + "MetricSetFields": { + "primaries": { + "docs": { + "count": 8030785507, + "deleted": 23527 + }, + "indexing": { + "index": { + "count": 56600767530, + "time": { + "ms": 5360597433 + } + } + }, + "search": { + "query": { + "count": 7898882196, + "time": { + "ms": 5848049597 + } + } + }, + "segments": { + "count": 11284, + "memory": { + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 1686972455265 + }, + "total_data_set_size": { + "bytes": 1686972455265 + } + } + }, + "total": { + "docs": { + "count": 8030785507, + "deleted": 23527 + }, + "indexing": { + "index": { + "count": 56600767530, + "time": { + "ms": 5360597433 + } + } + }, + "search": { + "query": { + "count": 7898882196, + "time": { + "ms": 5848049597 + } + } + }, + "segments": { + "count": 11284, + "memory": { + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 1686972455265 + }, + "total_data_set_size": { + "bytes": 1686972455265 + } + } + } + }, + "Index": ".monitoring-es-8-mb", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_7.17.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_7.17.json new file mode 100644 index 000000000000..997f5805c0be --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_7.17.json @@ -0,0 +1,97 @@ +{ + "RootFields": { + "service": { + "name": "elasticsearch" + } + }, + "ModuleFields": { + "cluster": { + "id": "1234", + "name": "helloworld" + } + }, + "MetricSetFields": { + "primaries": { + "docs": { + "count": 464, + "deleted": 252 + }, + "store": { + "size": { + "bytes": 85621934 + }, + "total_data_set_size": { + "bytes": 85621934 + } + }, + "indexing": { + "index": { + "count": 390, + "time": { + "ms": 1815 + } + } + }, + "search": { + "query": { + "count": 313, + "time": { + "ms": 806 + } + } + }, + "segments": { + "count": 80, + "memory": { + "bytes": 114024 + } + } + }, + "total": { + "docs": { + "count": 464, + "deleted": 252 + }, + "store": { + "size": { + "bytes": 85621934 + }, + "total_data_set_size": { + "bytes": 85621934 + } + }, + "indexing": { + "index": { + "count": 390, + "time": { + "ms": 1815 + } + } + }, + "search": { + "query": { + "count": 313, + "time": { + "ms": 806 + } + } + }, + "segments": { + "count": 80, + "memory": { + "bytes": 114024 + } + } + } + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_8.17.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_8.17.json new file mode 100644 index 000000000000..472f271b0403 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_8.17.json @@ -0,0 +1,97 @@ +{ + "RootFields": { + "service": { + "name": "elasticsearch" + } + }, + "ModuleFields": { + "cluster": { + "id": "1234", + "name": "helloworld" + } + }, + "MetricSetFields": { + "primaries": { + "docs": { + "count": 8030785507, + "deleted": 23527 + }, + "indexing": { + "index": { + "count": 56600767530, + "time": { + "ms": 5360597433 + } + } + }, + "search": { + "query": { + "count": 7898882196, + "time": { + "ms": 5848049597 + } + } + }, + "segments": { + "count": 11284, + "memory": { + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 1686972455265 + }, + "total_data_set_size": { + "bytes": 1686972455265 + } + } + }, + "total": { + "docs": { + "count": 8030785507, + "deleted": 23527 + }, + "indexing": { + "index": { + "count": 56600767530, + "time": { + "ms": 5360597433 + } + } + }, + "search": { + "query": { + "count": 7898882196, + "time": { + "ms": 5848049597 + } + } + }, + "segments": { + "count": 11284, + "memory": { + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 1686972455265 + }, + "total_data_set_size": { + "bytes": 1686972455265 + } + } + } + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_empty.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_empty.json new file mode 100644 index 000000000000..e166c1b38f70 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_empty.json @@ -0,0 +1,4 @@ +{ + "nodes": { + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717.json new file mode 100644 index 000000000000..af103a7260ef --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717.json @@ -0,0 +1,76 @@ +{ + "nodes" : { + "iyBjmE-eSviHPDdmjKXyeQ" : { + "indices" : { + "docs" : { + "count" : 18, + "deleted" : 119 + }, + "store" : { + "size_in_bytes" : 120986, + "total_data_set_size_in_bytes" : 120986 + }, + "indexing" : { + "index_total" : 135, + "index_time_in_millis" : 181 + }, + "search" : { + "query_total" : 28, + "query_time_in_millis" : 440 + }, + "segments" : { + "count" : 49, + "memory_in_bytes" : 32540 + } + } + }, + "vF3ak-83RKu_020pnVZJ_w" : { + "indices" : { + "docs" : { + "count" : 365, + "deleted" : 14 + }, + "store" : { + "size_in_bytes" : 42727990, + "total_data_set_size_in_bytes" : 42727990 + }, + "indexing" : { + "index_total" : 65, + "index_time_in_millis" : 959 + }, + "search" : { + "query_total" : 83, + "query_time_in_millis" : 95 + }, + "segments" : { + "count" : 12, + "memory_in_bytes" : 31024 + } + } + }, + "7-qiV2KLSSyZ74aKfCe6bg" : { + "indices" : { + "docs" : { + "count" : 81, + "deleted" : 119 + }, + "store" : { + "size_in_bytes" : 42772958, + "total_data_set_size_in_bytes" : 42772958 + }, + "indexing" : { + "index_total" : 190, + "index_time_in_millis" : 675 + }, + "search" : { + "query_total" : 202, + "query_time_in_millis" : 271 + }, + "segments" : { + "count" : 19, + "memory_in_bytes" : 50460 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717_field_as_string.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717_field_as_string.json new file mode 100644 index 000000000000..081662792593 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717_field_as_string.json @@ -0,0 +1,76 @@ +{ + "nodes" : { + "iyBjmE-eSviHPDdmjKXyeQ" : { + "indices" : { + "docs" : { + "count" : 18, + "deleted" : 119 + }, + "store" : { + "size_in_bytes" : 120986, + "total_data_set_size_in_bytes" : 120986 + }, + "indexing" : { + "index_total" : 135, + "index_time_in_millis" : 181 + }, + "search" : { + "query_total" : 28, + "query_time_in_millis" : 440 + }, + "segments" : { + "count" : 49, + "memory_in_bytes" : 32540 + } + } + }, + "vF3ak-83RKu_020pnVZJ_w" : { + "indices" : { + "docs" : { + "count" : 365, + "deleted" : 14 + }, + "store" : { + "size_in_bytes" : "42727990", + "total_data_set_size_in_bytes" : 42727990 + }, + "indexing" : { + "index_total" : 65, + "index_time_in_millis" : 959 + }, + "search" : { + "query_total" : 83, + "query_time_in_millis" : 95 + }, + "segments" : { + "count" : 12, + "memory_in_bytes" : 31024 + } + } + }, + "7-qiV2KLSSyZ74aKfCe6bg" : { + "indices" : { + "docs" : { + "count" : 81, + "deleted" : 119 + }, + "store" : { + "size_in_bytes" : 42772958, + "total_data_set_size_in_bytes" : 42772958 + }, + "indexing" : { + "index_total" : 190, + "index_time_in_millis" : 675 + }, + "search" : { + "query_total" : 202, + "query_time_in_millis" : 271 + }, + "segments" : { + "count" : 19, + "memory_in_bytes" : 50460 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817.json new file mode 100644 index 000000000000..d6969abd27d7 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817.json @@ -0,0 +1,164 @@ +{ + "nodes": { + "Hwq8Kg1eRNaFnFJKrKoqjA": { + "indices": { + "docs": { + "count": 1730135320, + "deleted": 0, + "total_size_in_bytes": 388140418237 + }, + "store": { + "size_in_bytes": 388140454876, + "total_data_set_size_in_bytes": 388140454876 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1595587408, + "query_time_in_millis": 461298337 + }, + "segments": { + "count": 91, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "ZmPOU7owRnOR4kTkRahVUg": { + "indices": { + "docs": { + "count": 1770383894, + "deleted": 0, + "total_size_in_bytes": 402572306308 + }, + "store": { + "size_in_bytes": 402572342569, + "total_data_set_size_in_bytes": 402572342569 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1468260676, + "query_time_in_millis": 417207984 + }, + "segments": { + "count": 90, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "5lbmiBszSxahvsDAs8jwUg": { + "indices": { + "docs": { + "count": 1422510642, + "deleted": 7521, + "total_size_in_bytes": 277974136298 + }, + "store": { + "size_in_bytes": 278444418475, + "total_data_set_size_in_bytes": 278444418475 + }, + "indexing": { + "index_total": 17581495402, + "index_time_in_millis": 1621090717 + }, + "search": { + "query_total": 1464783588, + "query_time_in_millis": 1634551313 + }, + "segments": { + "count": 3802, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 29982373, + "total_time_in_millis": 1658665229, + "total_size_in_bytes": 28791047343797, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 938940 + } + } + }, + "5rnp4uRrScygfEv1At_NuQ": { + "indices": { + "docs": { + "count": 1245022583, + "deleted": 15098, + "total_size_in_bytes": 246021449595 + }, + "store": { + "size_in_bytes": 246053805084, + "total_data_set_size_in_bytes": 246053805084 + }, + "indexing": { + "index_total": 19072011163, + "index_time_in_millis": 1839353503 + }, + "search": { + "query_total": 1717052931, + "query_time_in_millis": 1657131632 + }, + "segments": { + "count": 3411, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 31937349, + "total_time_in_millis": 1876634225, + "total_size_in_bytes": 31286022361033, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 968960 + } + } + }, + "E5Y9e0ZTRNmMhlL-XBPzcw": { + "indices": { + "docs": { + "count": 1862733068, + "deleted": 908, + "total_size_in_bytes": 371144474256 + }, + "store": { + "size_in_bytes": 371761434261, + "total_data_set_size_in_bytes": 371761434261 + }, + "indexing": { + "index_total": 19947260965, + "index_time_in_millis": 1900153213 + }, + "search": { + "query_total": 1653197593, + "query_time_in_millis": 1677860331 + }, + "segments": { + "count": 3890, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 34665061, + "total_time_in_millis": 1937237991, + "total_size_in_bytes": 32589352616465, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 922777 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_block.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_block.json new file mode 100644 index 000000000000..d7e5cdecd7b6 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_block.json @@ -0,0 +1,160 @@ +{ + "nodes": { + "Hwq8Kg1eRNaFnFJKrKoqjA": { + "indices": { + "docs": { + "count": 1730135320, + "deleted": 0, + "total_size_in_bytes": 388140418237 + }, + "store": { + "size_in_bytes": 388140454876, + "total_data_set_size_in_bytes": 388140454876 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1595587408, + "query_time_in_millis": 461298337 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "ZmPOU7owRnOR4kTkRahVUg": { + "indices": { + "docs": { + "count": 1770383894, + "deleted": 0, + "total_size_in_bytes": 402572306308 + }, + "store": { + "size_in_bytes": 402572342569, + "total_data_set_size_in_bytes": 402572342569 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1468260676, + "query_time_in_millis": 417207984 + }, + "segments": { + "count": 90, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "5lbmiBszSxahvsDAs8jwUg": { + "indices": { + "docs": { + "count": 1422510642, + "deleted": 7521, + "total_size_in_bytes": 277974136298 + }, + "store": { + "size_in_bytes": 278444418475, + "total_data_set_size_in_bytes": 278444418475 + }, + "indexing": { + "index_total": 17581495402, + "index_time_in_millis": 1621090717 + }, + "search": { + "query_total": 1464783588, + "query_time_in_millis": 1634551313 + }, + "segments": { + "count": 3802, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 29982373, + "total_time_in_millis": 1658665229, + "total_size_in_bytes": 28791047343797, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 938940 + } + } + }, + "5rnp4uRrScygfEv1At_NuQ": { + "indices": { + "docs": { + "count": 1245022583, + "deleted": 15098, + "total_size_in_bytes": 246021449595 + }, + "store": { + "size_in_bytes": 246053805084, + "total_data_set_size_in_bytes": 246053805084 + }, + "indexing": { + "index_total": 19072011163, + "index_time_in_millis": 1839353503 + }, + "search": { + "query_total": 1717052931, + "query_time_in_millis": 1657131632 + }, + "segments": { + "count": 3411, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 31937349, + "total_time_in_millis": 1876634225, + "total_size_in_bytes": 31286022361033, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 968960 + } + } + }, + "E5Y9e0ZTRNmMhlL-XBPzcw": { + "indices": { + "docs": { + "count": 1862733068, + "deleted": 908, + "total_size_in_bytes": 371144474256 + }, + "store": { + "size_in_bytes": 371761434261, + "total_data_set_size_in_bytes": 371761434261 + }, + "indexing": { + "index_total": 19947260965, + "index_time_in_millis": 1900153213 + }, + "search": { + "query_total": 1653197593, + "query_time_in_millis": 1677860331 + }, + "segments": { + "count": 3890, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 34665061, + "total_time_in_millis": 1937237991, + "total_size_in_bytes": 32589352616465, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 922777 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_fields.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_fields.json new file mode 100644 index 000000000000..7ee4284c9e76 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_fields.json @@ -0,0 +1,163 @@ +{ + "nodes": { + "Hwq8Kg1eRNaFnFJKrKoqjA": { + "indices": { + "docs": { + "deleted": 0, + "total_size_in_bytes": 388140418237 + }, + "store": { + "size_in_bytes": 388140454876, + "total_data_set_size_in_bytes": 388140454876 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1595587408, + "query_time_in_millis": 461298337 + }, + "segments": { + "count": 91, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "ZmPOU7owRnOR4kTkRahVUg": { + "indices": { + "docs": { + "count": 1770383894, + "deleted": 0, + "total_size_in_bytes": 402572306308 + }, + "store": { + "size_in_bytes": 402572342569, + "total_data_set_size_in_bytes": 402572342569 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1468260676, + "query_time_in_millis": 417207984 + }, + "segments": { + "count": 90, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "5lbmiBszSxahvsDAs8jwUg": { + "indices": { + "docs": { + "count": 1422510642, + "deleted": 7521, + "total_size_in_bytes": 277974136298 + }, + "store": { + "size_in_bytes": 278444418475, + "total_data_set_size_in_bytes": 278444418475 + }, + "indexing": { + "index_total": 17581495402, + "index_time_in_millis": 1621090717 + }, + "search": { + "query_total": 1464783588, + "query_time_in_millis": 1634551313 + }, + "segments": { + "count": 3802, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 29982373, + "total_time_in_millis": 1658665229, + "total_size_in_bytes": 28791047343797, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 938940 + } + } + }, + "5rnp4uRrScygfEv1At_NuQ": { + "indices": { + "docs": { + "count": 1245022583, + "deleted": 15098, + "total_size_in_bytes": 246021449595 + }, + "store": { + "size_in_bytes": 246053805084, + "total_data_set_size_in_bytes": 246053805084 + }, + "indexing": { + "index_total": 19072011163, + "index_time_in_millis": 1839353503 + }, + "search": { + "query_total": 1717052931, + "query_time_in_millis": 1657131632 + }, + "segments": { + "count": 3411, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 31937349, + "total_time_in_millis": 1876634225, + "total_size_in_bytes": 31286022361033, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 968960 + } + } + }, + "E5Y9e0ZTRNmMhlL-XBPzcw": { + "indices": { + "docs": { + "count": 1862733068, + "deleted": 908, + "total_size_in_bytes": 371144474256 + }, + "store": { + "size_in_bytes": 371761434261, + "total_data_set_size_in_bytes": 371761434261 + }, + "indexing": { + "index_total": 19947260965, + "index_time_in_millis": 1900153213 + }, + "search": { + "query_total": 1653197593, + "query_time_in_millis": 1677860331 + }, + "segments": { + "count": 3890, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 34665061, + "total_time_in_millis": 1937237991, + "total_size_in_bytes": 32589352616465, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 922777 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/data.go b/metricbeat/module/elasticsearch/index_summary/data.go index abd1d02d01df..e4a26d5e5488 100644 --- a/metricbeat/module/elasticsearch/index_summary/data.go +++ b/metricbeat/module/elasticsearch/index_summary/data.go @@ -30,82 +30,202 @@ import ( "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) -var ( - schema = s.Schema{ - "primaries": c.Dict("primaries", indexSummaryDict), - "total": c.Dict("total", indexSummaryDict), - } -) +type nodeStatsWrapper struct { + Nodes map[string]interface{} `json:"nodes"` +} -var indexSummaryDict = s.Schema{ - "docs": c.Dict("docs", s.Schema{ - "count": c.Int("count"), - "deleted": c.Int("deleted"), +var nodeItemSchema = s.Schema{ + "indices": c.Dict("indices", s.Schema{ + "docs": c.Dict("docs", s.Schema{ + "count": c.Int("count", s.Required), + "deleted": c.Int("deleted", s.Required), + }, c.DictRequired), + "store": c.Dict("store", s.Schema{ + "size": s.Object{ + "bytes": c.Int("size_in_bytes", s.Required), + }, + "total_data_set_size": s.Object{ + "bytes": c.Int("total_data_set_size_in_bytes", s.Required), + }, + }, c.DictRequired), + "indexing": c.Dict("indexing", s.Schema{ + "index": s.Object{ + "count": c.Int("index_total", s.Required), + "time": s.Object{ + "ms": c.Int("index_time_in_millis", s.Required), + }, + }, + }, c.DictRequired), + "search": c.Dict("search", s.Schema{ + "query": s.Object{ + "count": c.Int("query_total", s.Required), + "time": s.Object{ + "ms": c.Int("query_time_in_millis", s.Required), + }, + }, + }, c.DictRequired), + "segments": c.Dict("segments", s.Schema{ + "count": c.Int("count", s.Required), + "memory": s.Object{ + "bytes": c.Int("memory_in_bytes", s.Required), + }, + }, c.DictRequired), + "bulk": c.Dict("bulk", s.Schema{ + "operations": s.Object{ + "count": c.Int("total_operations", s.Required), + }, + "time": s.Object{ + "avg": s.Object{ + "bytes": c.Int("avg_size_in_bytes", s.Required), + }, + }, + "size": s.Object{ + "bytes": c.Int("total_size_in_bytes", s.Required), + }, + }, c.DictOptional), }), - "store": c.Dict("store", s.Schema{ - "size": s.Object{ - "bytes": c.Int("size_in_bytes"), - }, - "total_data_set_size": s.Object{ - "bytes": c.Int("total_data_set_size_in_bytes", s.Optional), - }, - }), - "segments": c.Dict("segments", s.Schema{ - "count": c.Int("count"), - "memory": s.Object{ - "bytes": c.Int("memory_in_bytes"), - }, - }), - "indexing": indexingDict, - "bulk": bulkStatsDict, - "search": searchDict, } -var indexingDict = c.Dict("indexing", s.Schema{ - "index": s.Object{ - "count": c.Int("index_total"), - "time": s.Object{ - "ms": c.Int("index_time_in_millis"), - }, - }, -}) - -var searchDict = c.Dict("search", s.Schema{ - "query": s.Object{ - "count": c.Int("query_total"), - "time": s.Object{ - "ms": c.Int("query_time_in_millis"), - }, - }, -}) - -var bulkStatsDict = c.Dict("bulk", s.Schema{ - "operations": s.Object{ - "count": c.Int("total_operations"), - }, - "time": s.Object{ - "avg": s.Object{ - "bytes": c.Int("avg_size_in_bytes"), - }, - }, - "size": s.Object{ - "bytes": c.Int("total_size_in_bytes"), - }, -}, c.DictOptional) - -func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXpack bool) error { - var all struct { - Data map[string]interface{} `json:"_all"` +type IndexSummaryMetricSet struct { + Primaries IndexSummary `json:"primaries"` + Total IndexSummary `json:"total"` +} + +type IndexSummary struct { + Docs DocsSection `json:"docs"` + Store StoreSection `json:"store"` + Indexing IndexingSection `json:"indexing"` + Search SearchSection `json:"search"` + Segments SegmentSection `json:"segments"` +} + +type DocsSection struct { + Count int64 `json:"count"` + Deleted int64 `json:"deleted"` +} + +type StoreSection struct { + Size struct { + Bytes int64 `json:"bytes"` + } `json:"size"` + TotalDataSetSize struct { + Bytes int64 `json:"bytes"` + } `json:"total_data_set_size"` +} + +type IndexingSection struct { + Index struct { + Count int64 `json:"count"` + Time struct { + Ms int64 `json:"ms"` + } `json:"time"` + } `json:"index"` +} + +type SearchSection struct { + Query struct { + Count int64 `json:"count"` + Time struct { + Ms int64 `json:"ms"` + } `json:"time"` + } `json:"query"` +} + +type SegmentSection struct { + Count int64 `json:"count"` + Memory struct { + Bytes int64 `json:"bytes"` + } `json:"memory"` +} + +func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXPack bool) error { + var wrapper nodeStatsWrapper + if err := json.Unmarshal(content, &wrapper); err != nil { + return fmt.Errorf("failure parsing NodeStats API response: %w", err) } - err := json.Unmarshal(content, &all) - if err != nil { - return fmt.Errorf("failure parsing Elasticsearch Stats API response: %w", err) + if len(wrapper.Nodes) == 0 { + return fmt.Errorf("no nodes found in NodeStats response") + } + + var total IndexSummary + for nodeKey, raw := range wrapper.Nodes { + err := addNodeMetrics(raw, &total) + if err != nil { + return fmt.Errorf("error processing node %q: %w", nodeKey, err) + } + } + + event := buildEvent(&info, &total, isXPack) + r.Event(event) + return nil +} + +func addNodeMetrics(rawNode interface{}, summary *IndexSummary) error { + nodeMap, ok := rawNode.(map[string]interface{}) + if !ok { + return fmt.Errorf("node is not a map") } - fields, err := schema.Apply(all.Data, s.FailOnRequired) + validated, err := nodeItemSchema.Apply(nodeMap, s.FailOnRequired) if err != nil { - return fmt.Errorf("failure applying stats schema: %w", err) + return err + } + + incrementValue := func(dst *int64, path ...string) { + if v, err := getInt64(validated, path...); err == nil { + *dst += v + } + } + + // Docs + incrementValue(&summary.Docs.Count, "indices", "docs", "count") + incrementValue(&summary.Docs.Deleted, "indices", "docs", "deleted") + + // Store + incrementValue(&summary.Store.Size.Bytes, "indices", "store", "size", "bytes") + incrementValue(&summary.Store.TotalDataSetSize.Bytes, "indices", "store", "total_data_set_size", "bytes") + + // Indexing + incrementValue(&summary.Indexing.Index.Count, "indices", "indexing", "index", "count") + incrementValue(&summary.Indexing.Index.Time.Ms, "indices", "indexing", "index", "time", "ms") + + // Search + incrementValue(&summary.Search.Query.Count, "indices", "search", "query", "count") + incrementValue(&summary.Search.Query.Time.Ms, "indices", "search", "query", "time", "ms") + + // Segments + incrementValue(&summary.Segments.Count, "indices", "segments", "count") + incrementValue(&summary.Segments.Memory.Bytes, "indices", "segments", "memory", "bytes") + + return nil +} + +func getInt64(m mapstr.M, path ...string) (int64, error) { + current := interface{}(m) + for _, key := range path { + mm, ok := current.(mapstr.M) + if !ok { + return 0, fmt.Errorf("expected mapstr.M at %q, got %T", key, current) + } + val, ok := mm[key] + if !ok { + return 0, fmt.Errorf("missing key: %q", key) + } + current = val + } + + i, ok := current.(int64) + if !ok { + return 0, fmt.Errorf("expected int64 at path %v, got %T", path, current) + } + return i, nil +} + +func buildEvent(info *elasticsearch.Info, summary *IndexSummary, isXPack bool) mb.Event { + eventNew := map[string]interface{}{ + "primaries": summary, + "total": summary, } var event mb.Event @@ -116,16 +236,12 @@ func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXp _, _ = event.ModuleFields.Put("cluster.name", info.ClusterName) _, _ = event.ModuleFields.Put("cluster.id", info.ClusterID) - event.MetricSetFields = fields + event.MetricSetFields = eventNew - // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` - // When using Agent, the index name is overwritten anyways. - if isXpack { + if isXPack { index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) event.Index = index } - r.Event(event) - - return nil + return event } diff --git a/metricbeat/module/elasticsearch/index_summary/data_test.go b/metricbeat/module/elasticsearch/index_summary/data_test.go index 3df0e61b2698..a43e151493bc 100644 --- a/metricbeat/module/elasticsearch/index_summary/data_test.go +++ b/metricbeat/module/elasticsearch/index_summary/data_test.go @@ -20,9 +20,9 @@ package index_summary import ( - "io/ioutil" "net/http" "net/http/httptest" + "os" "testing" "github.com/stretchr/testify/require" @@ -48,7 +48,7 @@ func createEsMuxer(license string) *http.ServeMux { http.NotFound(w, r) } - input, _ := ioutil.ReadFile("../index/_meta/test/root.710.json") + input, _ := os.ReadFile("../index/_meta/test/root.710.json") w.Write(input) } licenseHandler := func(w http.ResponseWriter, r *http.Request) { @@ -62,7 +62,7 @@ func createEsMuxer(license string) *http.ServeMux { mux.Handle("/_xpack/license", http.HandlerFunc(licenseHandler)) // for before 7.0 mux.Handle("/", http.HandlerFunc(rootHandler)) mux.Handle("/_stats", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - content, _ := ioutil.ReadFile("../index/_meta/test/stats.700-alpha1.json") + content, _ := os.ReadFile("../index/_meta/test/stats.700-alpha1.json") w.Write(content) })) @@ -82,17 +82,124 @@ func TestData(t *testing.T) { } func TestMapper(t *testing.T) { - elasticsearch.TestMapperWithInfo(t, "../index/_meta/test/stats.*.json", eventMapping) + elasticsearch.TestMapperWithInfo(t, "_meta/test/node_stats_v*17.json", eventMapping) +} + +func TestSummaryFromNodeStatsWithExpectedEventsV817(t *testing.T) { + elasticsearch.TestMapperWithExpectedEvents( + t, + "_meta/test/node_stats_v817.json", + []string{ + "_meta/test/expected_event_8.17.json", + }, + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + true, + eventMapping, + ) +} + +func TestSummaryMissingField(t *testing.T) { + elasticsearch.TestMapperExpectingError( + t, + "_meta/test/node_stats_v817_missing_fields.json", + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + true, + "error processing node \"Hwq8Kg1eRNaFnFJKrKoqjA\": key `indices.docs.count` not found", + eventMapping, + ) +} + +func TestSummaryMissingBlock(t *testing.T) { + elasticsearch.TestMapperExpectingError( + t, + "_meta/test/node_stats_v817_missing_block.json", + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + true, + "error processing node \"Hwq8Kg1eRNaFnFJKrKoqjA\": key `indices.segments` not found", + eventMapping, + ) +} + +func TestSummaryWrongFieldType_String(t *testing.T) { + elasticsearch.TestMapperExpectingError( + t, + "_meta/test/node_stats_v717_field_as_string.json", + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + true, + "error processing node \"vF3ak-83RKu_020pnVZJ_w\": wrong format in `indices.store.size_in_bytes`: expected integer, found string", + eventMapping, + ) +} + +func TestSummaryFromNodeStatsWithExpectedEventsV717(t *testing.T) { + elasticsearch.TestMapperWithExpectedEvents( + t, + "_meta/test/node_stats_v717.json", + []string{ + "_meta/test/expected_event_7.17.json", + }, + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + true, + eventMapping, + ) +} + +func TestSummaryFromNodeStatsWithExpectedEventsXPackV817(t *testing.T) { + elasticsearch.TestMapperWithExpectedEvents( + t, + "_meta/test/node_stats_v817.json", + []string{ + "_meta/test/expected_event_xpack_8.17.json", + }, + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + false, + eventMapping, + ) +} + +func TestSummaryFromNodeStatsWithExpectedEventsXPackV717(t *testing.T) { + elasticsearch.TestMapperWithExpectedEvents( + t, + "_meta/test/node_stats_v717.json", + []string{ + "_meta/test/expected_event_xpack_7.17.json", + }, + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + false, + eventMapping, + ) } func TestEmpty(t *testing.T) { - input, err := ioutil.ReadFile("../index/_meta/test/empty.512.json") - require.NoError(t, err) + input, errReading := os.ReadFile("_meta/test/node_stats_empty.json") + require.NoError(t, errReading) reporter := &mbtest.CapturingReporterV2{} - eventMapping(reporter, info, input, true) - require.Empty(t, reporter.GetErrors()) - require.Equal(t, 1, len(reporter.GetEvents())) + err := eventMapping(reporter, info, input, true) + + require.ErrorContains(t, err, "no nodes found in NodeStats response") + require.Equal(t, 0, len(reporter.GetEvents())) } func getConfig(host string) map[string]interface{} { diff --git a/metricbeat/module/elasticsearch/index_summary/index_summary.go b/metricbeat/module/elasticsearch/index_summary/index_summary.go index d3a91b442479..3c1c4a153abd 100644 --- a/metricbeat/module/elasticsearch/index_summary/index_summary.go +++ b/metricbeat/module/elasticsearch/index_summary/index_summary.go @@ -24,8 +24,6 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" - - "github.com/elastic/elastic-agent-libs/version" ) // init registers the MetricSet with the central registry. @@ -38,9 +36,13 @@ func init() { } const ( - statsPath = "/_stats" + nodeStatsPath = "/_nodes/stats" +<<<<<<< HEAD allowClosedIndices = "forbid_closed_indices=false" +======= + nodeStatsParameters = "level=node&filter_path=nodes.*.indices.docs,nodes.*.indices.indexing.index_total,nodes.*.indices.indexing.index_time_in_millis,nodes.*.indices.search.query_total,nodes.*.indices.search.query_time_in_millis,nodes.*.indices.segments.count,nodes.*.indices.segments.memory_in_bytes,nodes.*.indices.store.size_in_bytes,nodes.*.indices.store.total_data_set_size_in_bytes" +>>>>>>> 63a537a17 ([MetricBeat][StackMonitoring] fetching index summary from node/_stats endpoint (#45049)) ) var ( @@ -58,7 +60,7 @@ type MetricSet struct { // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Get the stats from the local node - ms, err := elasticsearch.NewMetricSet(base, statsPath) + ms, err := elasticsearch.NewMetricSet(base, nodeStatsPath) if err != nil { return nil, err } @@ -75,16 +77,16 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return nil } - info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+statsPath) + info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+nodeStatsPath) if err != nil { return fmt.Errorf("failed to get info from Elasticsearch: %w", err) } - if err := m.updateServicePath(*info.Version.Number); err != nil { + if err := m.updateServicePath(); err != nil { return err } - content, err := m.HTTP.FetchContent() + content, err := m.FetchContent() if err != nil { return err } @@ -92,8 +94,8 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return eventMapping(r, info, content, m.XPackEnabled) } -func (m *MetricSet) updateServicePath(esVersion version.V) error { - p, err := getServicePath(esVersion) +func (m *MetricSet) updateServicePath() error { + p, err := getServicePath() if err != nil { return err } @@ -102,16 +104,20 @@ func (m *MetricSet) updateServicePath(esVersion version.V) error { return nil } -func getServicePath(esVersion version.V) (string, error) { - currPath := statsPath +func getServicePath() (string, error) { + currPath := nodeStatsPath u, err := url.Parse(currPath) if err != nil { return "", err } +<<<<<<< HEAD if !esVersion.LessThan(elasticsearch.BulkStatsAvailableVersion) { u.RawQuery += allowClosedIndices } +======= + u.RawQuery += nodeStatsParameters +>>>>>>> 63a537a17 ([MetricBeat][StackMonitoring] fetching index summary from node/_stats endpoint (#45049)) return u.String(), nil } diff --git a/metricbeat/module/elasticsearch/index_summary/index_summary_test.go b/metricbeat/module/elasticsearch/index_summary/index_summary_test.go new file mode 100644 index 000000000000..e61d26db4304 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/index_summary_test.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !integration + +package index_summary + +import ( + "testing" +) + +func TestGetServicePath(t *testing.T) { + expectedPath := "/_nodes/stats?level=node&filter_path=nodes.*.indices.docs,nodes.*.indices.indexing.index_total,nodes.*.indices.indexing.index_time_in_millis,nodes.*.indices.search.query_total,nodes.*.indices.search.query_time_in_millis,nodes.*.indices.segments.count,nodes.*.indices.segments.memory_in_bytes,nodes.*.indices.store.size_in_bytes,nodes.*.indices.store.total_data_set_size_in_bytes" + path, err := getServicePath() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if path != expectedPath { + t.Errorf("expected path %q, got %q", expectedPath, path) + } +} diff --git a/metricbeat/module/elasticsearch/testing.go b/metricbeat/module/elasticsearch/testing.go index 63f041c44fd5..b0fe6748c461 100644 --- a/metricbeat/module/elasticsearch/testing.go +++ b/metricbeat/module/elasticsearch/testing.go @@ -20,10 +20,14 @@ package elasticsearch import ( - "io/ioutil" + "encoding/json" + "fmt" + "os" "path/filepath" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/metricbeat/helper" @@ -41,7 +45,7 @@ func TestMapper(t *testing.T, glob string, mapper func(mb.ReporterV2, []byte) er for _, f := range files { t.Run(f, func(t *testing.T) { - input, err := ioutil.ReadFile(f) + input, err := os.ReadFile(f) require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} @@ -58,7 +62,7 @@ func TestMapperWithInfo(t *testing.T, glob string, mapper func(mb.ReporterV2, In files, err := filepath.Glob(glob) require.NoError(t, err) // Makes sure glob matches at least 1 file - require.True(t, len(files) > 0) + require.True(t, len(files) > 0, "Glob should match at least one file") info := Info{ ClusterID: "1234", @@ -67,18 +71,85 @@ func TestMapperWithInfo(t *testing.T, glob string, mapper func(mb.ReporterV2, In for _, f := range files { t.Run(f, func(t *testing.T) { - input, err := ioutil.ReadFile(f) + input, err := os.ReadFile(f) require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} err = mapper(reporter, info, input, true) require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) require.Equal(t, 0, len(reporter.GetErrors())) }) } } +func TestMapperWithExpectedEvents( + t *testing.T, + inputPath string, + expectedFiles []string, + info Info, + isXPack bool, + mapper func(mb.ReporterV2, Info, []byte, bool) error, +) { + input, err := os.ReadFile(inputPath) + require.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + err = mapper(reporter, info, input, isXPack) + require.NoError(t, err) + + events := reporter.GetEvents() + + expected := loadExpectedEventsFromFiles(t, expectedFiles) + require.Equal(t, len(expected), len(events), "Number of events mismatch") + + for i, ev := range events { + actualBytes, err := json.Marshal(ev) + require.NoError(t, err) + + var actual map[string]interface{} + err = json.Unmarshal(actualBytes, &actual) + require.NoError(t, err) + + assert.Equal(t, expected[i], actual, fmt.Sprintf("Mismatch in event #%d", i)) + } +} + +func TestMapperExpectingError( + t *testing.T, + inputPath string, + info Info, + isXPack bool, + errorMessage string, + mapper func(mb.ReporterV2, Info, []byte, bool) error, +) { + input, err := os.ReadFile(inputPath) + require.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + err = mapper(reporter, info, input, isXPack) + require.ErrorContains(t, err, errorMessage) + + events := reporter.GetEvents() + require.Equal(t, 0, len(events), "Number of events mismatch") +} + +func loadExpectedEventsFromFiles(t *testing.T, files []string) []map[string]interface{} { + expected := make([]map[string]interface{}, 0, len(files)) + for _, f := range files { + content, err := os.ReadFile(f) + require.NoError(t, err) + + var ev map[string]interface{} + err = json.Unmarshal(content, &ev) + require.NoError(t, err) + + expected = append(expected, ev) + } + return expected +} + // TestMapperWithMetricSetAndInfo tests mapping methods with Info fields func TestMapperWithMetricSetAndInfo(t *testing.T, glob string, ms MetricSetAPI, mapper func(mb.ReporterV2, MetricSetAPI, Info, []byte, bool) error) { files, err := filepath.Glob(glob) @@ -93,7 +164,7 @@ func TestMapperWithMetricSetAndInfo(t *testing.T, glob string, ms MetricSetAPI, for _, f := range files { t.Run(f, func(t *testing.T) { - input, err := ioutil.ReadFile(f) + input, err := os.ReadFile(f) require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} @@ -125,7 +196,7 @@ func TestMapperWithHttpHelper(t *testing.T, glob string, httpClient *helper.HTTP for _, f := range files { t.Run(f, func(t *testing.T) { - input, err := ioutil.ReadFile(f) + input, err := os.ReadFile(f) require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} From 148add703d42cc07a45488201911d4771df04866 Mon Sep 17 00:00:00 2001 From: Chris Earle Date: Fri, 4 Jul 2025 13:55:32 -0600 Subject: [PATCH 2/3] Fix merge conflict (accept new changes exclusively) --- .../elasticsearch/index_summary/index_summary.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/metricbeat/module/elasticsearch/index_summary/index_summary.go b/metricbeat/module/elasticsearch/index_summary/index_summary.go index 3c1c4a153abd..105c179b1b17 100644 --- a/metricbeat/module/elasticsearch/index_summary/index_summary.go +++ b/metricbeat/module/elasticsearch/index_summary/index_summary.go @@ -38,11 +38,7 @@ func init() { const ( nodeStatsPath = "/_nodes/stats" -<<<<<<< HEAD - allowClosedIndices = "forbid_closed_indices=false" -======= nodeStatsParameters = "level=node&filter_path=nodes.*.indices.docs,nodes.*.indices.indexing.index_total,nodes.*.indices.indexing.index_time_in_millis,nodes.*.indices.search.query_total,nodes.*.indices.search.query_time_in_millis,nodes.*.indices.segments.count,nodes.*.indices.segments.memory_in_bytes,nodes.*.indices.store.size_in_bytes,nodes.*.indices.store.total_data_set_size_in_bytes" ->>>>>>> 63a537a17 ([MetricBeat][StackMonitoring] fetching index summary from node/_stats endpoint (#45049)) ) var ( @@ -111,13 +107,7 @@ func getServicePath() (string, error) { return "", err } -<<<<<<< HEAD - if !esVersion.LessThan(elasticsearch.BulkStatsAvailableVersion) { - u.RawQuery += allowClosedIndices - } -======= u.RawQuery += nodeStatsParameters ->>>>>>> 63a537a17 ([MetricBeat][StackMonitoring] fetching index summary from node/_stats endpoint (#45049)) return u.String(), nil } From b4492c330e257d39fd3d8a2e287432760259cef1 Mon Sep 17 00:00:00 2001 From: Chris Earle Date: Fri, 4 Jul 2025 14:17:50 -0600 Subject: [PATCH 3/3] Update test to use multierr for older versions --- metricbeat/module/elasticsearch/index_summary/data_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/elasticsearch/index_summary/data_test.go b/metricbeat/module/elasticsearch/index_summary/data_test.go index a43e151493bc..9e8502049924 100644 --- a/metricbeat/module/elasticsearch/index_summary/data_test.go +++ b/metricbeat/module/elasticsearch/index_summary/data_test.go @@ -110,7 +110,7 @@ func TestSummaryMissingField(t *testing.T) { ClusterName: "helloworld", }, true, - "error processing node \"Hwq8Kg1eRNaFnFJKrKoqjA\": key `indices.docs.count` not found", + "error processing node \"Hwq8Kg1eRNaFnFJKrKoqjA\": 1 error: key `indices.docs.count` not found", eventMapping, ) } @@ -124,7 +124,7 @@ func TestSummaryMissingBlock(t *testing.T) { ClusterName: "helloworld", }, true, - "error processing node \"Hwq8Kg1eRNaFnFJKrKoqjA\": key `indices.segments` not found", + "error processing node \"Hwq8Kg1eRNaFnFJKrKoqjA\": 1 error: key `indices.segments` not found", eventMapping, ) } @@ -138,7 +138,7 @@ func TestSummaryWrongFieldType_String(t *testing.T) { ClusterName: "helloworld", }, true, - "error processing node \"vF3ak-83RKu_020pnVZJ_w\": wrong format in `indices.store.size_in_bytes`: expected integer, found string", + "error processing node \"vF3ak-83RKu_020pnVZJ_w\": 1 error: wrong format in `indices.store.size_in_bytes`: expected integer, found string", eventMapping, ) }