diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 39eb6789b455..34dc4edc90d2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -62,6 +62,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] *Metricbeat* +- Change index summary metricset to use `_nodes/stats` API instead of `_stats` API to avoid data gaps. {pull}45049[45049] - Add support for `_nodes/stats` URIs that work with legacy versions of Elasticsearch {pull}44307[44307] - Setting period for counter cache for Prometheus remote_write at least to 60sec {pull}38553[38553] - Remove fallback to the node limit for the `kubernetes.pod.cpu.usage.limit.pct` and `kubernetes.pod.memory.usage.limit.pct` metrics calculation diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_7.17.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_7.17.json new file mode 100644 index 000000000000..e622a4f26b2d --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_7.17.json @@ -0,0 +1,97 @@ +{ + "RootFields": { + "service": { + "name": "elasticsearch" + } + }, + "ModuleFields": { + "cluster": { + "id": "1234", + "name": "helloworld" + } + }, + "MetricSetFields": { + "primaries": { + "docs": { + "count": 464, + "deleted": 252 + }, + "store": { + "size": { + "bytes": 85621934 + }, + "total_data_set_size": { + "bytes": 85621934 + } + }, + "indexing": { + "index": { + "count": 390, + "time": { + "ms": 1815 + } + } + }, + "search": { + "query": { + "count": 313, + "time": { + "ms": 806 + } + } + }, + "segments": { + "count": 80, + "memory": { + "bytes": 114024 + } + } + }, + "total": { + "docs": { + "count": 464, + "deleted": 252 + }, + "store": { + "size": { + "bytes": 85621934 + }, + "total_data_set_size": { + "bytes": 85621934 + } + }, + "indexing": { + "index": { + "count": 390, + "time": { + "ms": 1815 + } + } + }, + "search": { + "query": { + "count": 313, + "time": { + "ms": 806 + } + } + }, + "segments": { + "count": 80, + "memory": { + "bytes": 114024 + } + } + } + }, + "Index": ".monitoring-es-8-mb", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_8.17.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_8.17.json new file mode 100644 index 000000000000..82d3cf3778d0 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_8.17.json @@ -0,0 +1,97 @@ +{ + "RootFields": { + "service": { + "name": "elasticsearch" + } + }, + "ModuleFields": { + "cluster": { + "id": "1234", + "name": "helloworld" + } + }, + "MetricSetFields": { + "primaries": { + "docs": { + "count": 8030785507, + "deleted": 23527 + }, + "indexing": { + "index": { + "count": 56600767530, + "time": { + "ms": 5360597433 + } + } + }, + "search": { + "query": { + "count": 7898882196, + "time": { + "ms": 5848049597 + } + } + }, + "segments": { + "count": 11284, + "memory": { + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 1686972455265 + }, + "total_data_set_size": { + "bytes": 1686972455265 + } + } + }, + "total": { + "docs": { + "count": 8030785507, + "deleted": 23527 + }, + "indexing": { + "index": { + "count": 56600767530, + "time": { + "ms": 5360597433 + } + } + }, + "search": { + "query": { + "count": 7898882196, + "time": { + "ms": 5848049597 + } + } + }, + "segments": { + "count": 11284, + "memory": { + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 1686972455265 + }, + "total_data_set_size": { + "bytes": 1686972455265 + } + } + } + }, + "Index": ".monitoring-es-8-mb", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_7.17.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_7.17.json new file mode 100644 index 000000000000..997f5805c0be --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_7.17.json @@ -0,0 +1,97 @@ +{ + "RootFields": { + "service": { + "name": "elasticsearch" + } + }, + "ModuleFields": { + "cluster": { + "id": "1234", + "name": "helloworld" + } + }, + "MetricSetFields": { + "primaries": { + "docs": { + "count": 464, + "deleted": 252 + }, + "store": { + "size": { + "bytes": 85621934 + }, + "total_data_set_size": { + "bytes": 85621934 + } + }, + "indexing": { + "index": { + "count": 390, + "time": { + "ms": 1815 + } + } + }, + "search": { + "query": { + "count": 313, + "time": { + "ms": 806 + } + } + }, + "segments": { + "count": 80, + "memory": { + "bytes": 114024 + } + } + }, + "total": { + "docs": { + "count": 464, + "deleted": 252 + }, + "store": { + "size": { + "bytes": 85621934 + }, + "total_data_set_size": { + "bytes": 85621934 + } + }, + "indexing": { + "index": { + "count": 390, + "time": { + "ms": 1815 + } + } + }, + "search": { + "query": { + "count": 313, + "time": { + "ms": 806 + } + } + }, + "segments": { + "count": 80, + "memory": { + "bytes": 114024 + } + } + } + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_8.17.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_8.17.json new file mode 100644 index 000000000000..472f271b0403 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/expected_event_xpack_8.17.json @@ -0,0 +1,97 @@ +{ + "RootFields": { + "service": { + "name": "elasticsearch" + } + }, + "ModuleFields": { + "cluster": { + "id": "1234", + "name": "helloworld" + } + }, + "MetricSetFields": { + "primaries": { + "docs": { + "count": 8030785507, + "deleted": 23527 + }, + "indexing": { + "index": { + "count": 56600767530, + "time": { + "ms": 5360597433 + } + } + }, + "search": { + "query": { + "count": 7898882196, + "time": { + "ms": 5848049597 + } + } + }, + "segments": { + "count": 11284, + "memory": { + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 1686972455265 + }, + "total_data_set_size": { + "bytes": 1686972455265 + } + } + }, + "total": { + "docs": { + "count": 8030785507, + "deleted": 23527 + }, + "indexing": { + "index": { + "count": 56600767530, + "time": { + "ms": 5360597433 + } + } + }, + "search": { + "query": { + "count": 7898882196, + "time": { + "ms": 5848049597 + } + } + }, + "segments": { + "count": 11284, + "memory": { + "bytes": 0 + } + }, + "store": { + "size": { + "bytes": 1686972455265 + }, + "total_data_set_size": { + "bytes": 1686972455265 + } + } + } + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_empty.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_empty.json new file mode 100644 index 000000000000..e166c1b38f70 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_empty.json @@ -0,0 +1,4 @@ +{ + "nodes": { + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717.json new file mode 100644 index 000000000000..af103a7260ef --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717.json @@ -0,0 +1,76 @@ +{ + "nodes" : { + "iyBjmE-eSviHPDdmjKXyeQ" : { + "indices" : { + "docs" : { + "count" : 18, + "deleted" : 119 + }, + "store" : { + "size_in_bytes" : 120986, + "total_data_set_size_in_bytes" : 120986 + }, + "indexing" : { + "index_total" : 135, + "index_time_in_millis" : 181 + }, + "search" : { + "query_total" : 28, + "query_time_in_millis" : 440 + }, + "segments" : { + "count" : 49, + "memory_in_bytes" : 32540 + } + } + }, + "vF3ak-83RKu_020pnVZJ_w" : { + "indices" : { + "docs" : { + "count" : 365, + "deleted" : 14 + }, + "store" : { + "size_in_bytes" : 42727990, + "total_data_set_size_in_bytes" : 42727990 + }, + "indexing" : { + "index_total" : 65, + "index_time_in_millis" : 959 + }, + "search" : { + "query_total" : 83, + "query_time_in_millis" : 95 + }, + "segments" : { + "count" : 12, + "memory_in_bytes" : 31024 + } + } + }, + "7-qiV2KLSSyZ74aKfCe6bg" : { + "indices" : { + "docs" : { + "count" : 81, + "deleted" : 119 + }, + "store" : { + "size_in_bytes" : 42772958, + "total_data_set_size_in_bytes" : 42772958 + }, + "indexing" : { + "index_total" : 190, + "index_time_in_millis" : 675 + }, + "search" : { + "query_total" : 202, + "query_time_in_millis" : 271 + }, + "segments" : { + "count" : 19, + "memory_in_bytes" : 50460 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717_field_as_string.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717_field_as_string.json new file mode 100644 index 000000000000..081662792593 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v717_field_as_string.json @@ -0,0 +1,76 @@ +{ + "nodes" : { + "iyBjmE-eSviHPDdmjKXyeQ" : { + "indices" : { + "docs" : { + "count" : 18, + "deleted" : 119 + }, + "store" : { + "size_in_bytes" : 120986, + "total_data_set_size_in_bytes" : 120986 + }, + "indexing" : { + "index_total" : 135, + "index_time_in_millis" : 181 + }, + "search" : { + "query_total" : 28, + "query_time_in_millis" : 440 + }, + "segments" : { + "count" : 49, + "memory_in_bytes" : 32540 + } + } + }, + "vF3ak-83RKu_020pnVZJ_w" : { + "indices" : { + "docs" : { + "count" : 365, + "deleted" : 14 + }, + "store" : { + "size_in_bytes" : "42727990", + "total_data_set_size_in_bytes" : 42727990 + }, + "indexing" : { + "index_total" : 65, + "index_time_in_millis" : 959 + }, + "search" : { + "query_total" : 83, + "query_time_in_millis" : 95 + }, + "segments" : { + "count" : 12, + "memory_in_bytes" : 31024 + } + } + }, + "7-qiV2KLSSyZ74aKfCe6bg" : { + "indices" : { + "docs" : { + "count" : 81, + "deleted" : 119 + }, + "store" : { + "size_in_bytes" : 42772958, + "total_data_set_size_in_bytes" : 42772958 + }, + "indexing" : { + "index_total" : 190, + "index_time_in_millis" : 675 + }, + "search" : { + "query_total" : 202, + "query_time_in_millis" : 271 + }, + "segments" : { + "count" : 19, + "memory_in_bytes" : 50460 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817.json new file mode 100644 index 000000000000..d6969abd27d7 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817.json @@ -0,0 +1,164 @@ +{ + "nodes": { + "Hwq8Kg1eRNaFnFJKrKoqjA": { + "indices": { + "docs": { + "count": 1730135320, + "deleted": 0, + "total_size_in_bytes": 388140418237 + }, + "store": { + "size_in_bytes": 388140454876, + "total_data_set_size_in_bytes": 388140454876 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1595587408, + "query_time_in_millis": 461298337 + }, + "segments": { + "count": 91, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "ZmPOU7owRnOR4kTkRahVUg": { + "indices": { + "docs": { + "count": 1770383894, + "deleted": 0, + "total_size_in_bytes": 402572306308 + }, + "store": { + "size_in_bytes": 402572342569, + "total_data_set_size_in_bytes": 402572342569 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1468260676, + "query_time_in_millis": 417207984 + }, + "segments": { + "count": 90, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "5lbmiBszSxahvsDAs8jwUg": { + "indices": { + "docs": { + "count": 1422510642, + "deleted": 7521, + "total_size_in_bytes": 277974136298 + }, + "store": { + "size_in_bytes": 278444418475, + "total_data_set_size_in_bytes": 278444418475 + }, + "indexing": { + "index_total": 17581495402, + "index_time_in_millis": 1621090717 + }, + "search": { + "query_total": 1464783588, + "query_time_in_millis": 1634551313 + }, + "segments": { + "count": 3802, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 29982373, + "total_time_in_millis": 1658665229, + "total_size_in_bytes": 28791047343797, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 938940 + } + } + }, + "5rnp4uRrScygfEv1At_NuQ": { + "indices": { + "docs": { + "count": 1245022583, + "deleted": 15098, + "total_size_in_bytes": 246021449595 + }, + "store": { + "size_in_bytes": 246053805084, + "total_data_set_size_in_bytes": 246053805084 + }, + "indexing": { + "index_total": 19072011163, + "index_time_in_millis": 1839353503 + }, + "search": { + "query_total": 1717052931, + "query_time_in_millis": 1657131632 + }, + "segments": { + "count": 3411, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 31937349, + "total_time_in_millis": 1876634225, + "total_size_in_bytes": 31286022361033, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 968960 + } + } + }, + "E5Y9e0ZTRNmMhlL-XBPzcw": { + "indices": { + "docs": { + "count": 1862733068, + "deleted": 908, + "total_size_in_bytes": 371144474256 + }, + "store": { + "size_in_bytes": 371761434261, + "total_data_set_size_in_bytes": 371761434261 + }, + "indexing": { + "index_total": 19947260965, + "index_time_in_millis": 1900153213 + }, + "search": { + "query_total": 1653197593, + "query_time_in_millis": 1677860331 + }, + "segments": { + "count": 3890, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 34665061, + "total_time_in_millis": 1937237991, + "total_size_in_bytes": 32589352616465, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 922777 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_block.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_block.json new file mode 100644 index 000000000000..d7e5cdecd7b6 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_block.json @@ -0,0 +1,160 @@ +{ + "nodes": { + "Hwq8Kg1eRNaFnFJKrKoqjA": { + "indices": { + "docs": { + "count": 1730135320, + "deleted": 0, + "total_size_in_bytes": 388140418237 + }, + "store": { + "size_in_bytes": 388140454876, + "total_data_set_size_in_bytes": 388140454876 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1595587408, + "query_time_in_millis": 461298337 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "ZmPOU7owRnOR4kTkRahVUg": { + "indices": { + "docs": { + "count": 1770383894, + "deleted": 0, + "total_size_in_bytes": 402572306308 + }, + "store": { + "size_in_bytes": 402572342569, + "total_data_set_size_in_bytes": 402572342569 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1468260676, + "query_time_in_millis": 417207984 + }, + "segments": { + "count": 90, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "5lbmiBszSxahvsDAs8jwUg": { + "indices": { + "docs": { + "count": 1422510642, + "deleted": 7521, + "total_size_in_bytes": 277974136298 + }, + "store": { + "size_in_bytes": 278444418475, + "total_data_set_size_in_bytes": 278444418475 + }, + "indexing": { + "index_total": 17581495402, + "index_time_in_millis": 1621090717 + }, + "search": { + "query_total": 1464783588, + "query_time_in_millis": 1634551313 + }, + "segments": { + "count": 3802, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 29982373, + "total_time_in_millis": 1658665229, + "total_size_in_bytes": 28791047343797, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 938940 + } + } + }, + "5rnp4uRrScygfEv1At_NuQ": { + "indices": { + "docs": { + "count": 1245022583, + "deleted": 15098, + "total_size_in_bytes": 246021449595 + }, + "store": { + "size_in_bytes": 246053805084, + "total_data_set_size_in_bytes": 246053805084 + }, + "indexing": { + "index_total": 19072011163, + "index_time_in_millis": 1839353503 + }, + "search": { + "query_total": 1717052931, + "query_time_in_millis": 1657131632 + }, + "segments": { + "count": 3411, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 31937349, + "total_time_in_millis": 1876634225, + "total_size_in_bytes": 31286022361033, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 968960 + } + } + }, + "E5Y9e0ZTRNmMhlL-XBPzcw": { + "indices": { + "docs": { + "count": 1862733068, + "deleted": 908, + "total_size_in_bytes": 371144474256 + }, + "store": { + "size_in_bytes": 371761434261, + "total_data_set_size_in_bytes": 371761434261 + }, + "indexing": { + "index_total": 19947260965, + "index_time_in_millis": 1900153213 + }, + "search": { + "query_total": 1653197593, + "query_time_in_millis": 1677860331 + }, + "segments": { + "count": 3890, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 34665061, + "total_time_in_millis": 1937237991, + "total_size_in_bytes": 32589352616465, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 922777 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_fields.json b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_fields.json new file mode 100644 index 000000000000..7ee4284c9e76 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/test/node_stats_v817_missing_fields.json @@ -0,0 +1,163 @@ +{ + "nodes": { + "Hwq8Kg1eRNaFnFJKrKoqjA": { + "indices": { + "docs": { + "deleted": 0, + "total_size_in_bytes": 388140418237 + }, + "store": { + "size_in_bytes": 388140454876, + "total_data_set_size_in_bytes": 388140454876 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1595587408, + "query_time_in_millis": 461298337 + }, + "segments": { + "count": 91, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "ZmPOU7owRnOR4kTkRahVUg": { + "indices": { + "docs": { + "count": 1770383894, + "deleted": 0, + "total_size_in_bytes": 402572306308 + }, + "store": { + "size_in_bytes": 402572342569, + "total_data_set_size_in_bytes": 402572342569 + }, + "indexing": { + "index_total": 0, + "index_time_in_millis": 0 + }, + "search": { + "query_total": 1468260676, + "query_time_in_millis": 417207984 + }, + "segments": { + "count": 90, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 0, + "total_time_in_millis": 0, + "total_size_in_bytes": 0, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 0 + } + } + }, + "5lbmiBszSxahvsDAs8jwUg": { + "indices": { + "docs": { + "count": 1422510642, + "deleted": 7521, + "total_size_in_bytes": 277974136298 + }, + "store": { + "size_in_bytes": 278444418475, + "total_data_set_size_in_bytes": 278444418475 + }, + "indexing": { + "index_total": 17581495402, + "index_time_in_millis": 1621090717 + }, + "search": { + "query_total": 1464783588, + "query_time_in_millis": 1634551313 + }, + "segments": { + "count": 3802, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 29982373, + "total_time_in_millis": 1658665229, + "total_size_in_bytes": 28791047343797, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 938940 + } + } + }, + "5rnp4uRrScygfEv1At_NuQ": { + "indices": { + "docs": { + "count": 1245022583, + "deleted": 15098, + "total_size_in_bytes": 246021449595 + }, + "store": { + "size_in_bytes": 246053805084, + "total_data_set_size_in_bytes": 246053805084 + }, + "indexing": { + "index_total": 19072011163, + "index_time_in_millis": 1839353503 + }, + "search": { + "query_total": 1717052931, + "query_time_in_millis": 1657131632 + }, + "segments": { + "count": 3411, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 31937349, + "total_time_in_millis": 1876634225, + "total_size_in_bytes": 31286022361033, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 968960 + } + } + }, + "E5Y9e0ZTRNmMhlL-XBPzcw": { + "indices": { + "docs": { + "count": 1862733068, + "deleted": 908, + "total_size_in_bytes": 371144474256 + }, + "store": { + "size_in_bytes": 371761434261, + "total_data_set_size_in_bytes": 371761434261 + }, + "indexing": { + "index_total": 19947260965, + "index_time_in_millis": 1900153213 + }, + "search": { + "query_total": 1653197593, + "query_time_in_millis": 1677860331 + }, + "segments": { + "count": 3890, + "memory_in_bytes": 0 + }, + "bulk": { + "total_operations": 34665061, + "total_time_in_millis": 1937237991, + "total_size_in_bytes": 32589352616465, + "avg_time_in_millis": 0, + "avg_size_in_bytes": 922777 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/data.go b/metricbeat/module/elasticsearch/index_summary/data.go index abd1d02d01df..e4a26d5e5488 100644 --- a/metricbeat/module/elasticsearch/index_summary/data.go +++ b/metricbeat/module/elasticsearch/index_summary/data.go @@ -30,82 +30,202 @@ import ( "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) -var ( - schema = s.Schema{ - "primaries": c.Dict("primaries", indexSummaryDict), - "total": c.Dict("total", indexSummaryDict), - } -) +type nodeStatsWrapper struct { + Nodes map[string]interface{} `json:"nodes"` +} -var indexSummaryDict = s.Schema{ - "docs": c.Dict("docs", s.Schema{ - "count": c.Int("count"), - "deleted": c.Int("deleted"), +var nodeItemSchema = s.Schema{ + "indices": c.Dict("indices", s.Schema{ + "docs": c.Dict("docs", s.Schema{ + "count": c.Int("count", s.Required), + "deleted": c.Int("deleted", s.Required), + }, c.DictRequired), + "store": c.Dict("store", s.Schema{ + "size": s.Object{ + "bytes": c.Int("size_in_bytes", s.Required), + }, + "total_data_set_size": s.Object{ + "bytes": c.Int("total_data_set_size_in_bytes", s.Required), + }, + }, c.DictRequired), + "indexing": c.Dict("indexing", s.Schema{ + "index": s.Object{ + "count": c.Int("index_total", s.Required), + "time": s.Object{ + "ms": c.Int("index_time_in_millis", s.Required), + }, + }, + }, c.DictRequired), + "search": c.Dict("search", s.Schema{ + "query": s.Object{ + "count": c.Int("query_total", s.Required), + "time": s.Object{ + "ms": c.Int("query_time_in_millis", s.Required), + }, + }, + }, c.DictRequired), + "segments": c.Dict("segments", s.Schema{ + "count": c.Int("count", s.Required), + "memory": s.Object{ + "bytes": c.Int("memory_in_bytes", s.Required), + }, + }, c.DictRequired), + "bulk": c.Dict("bulk", s.Schema{ + "operations": s.Object{ + "count": c.Int("total_operations", s.Required), + }, + "time": s.Object{ + "avg": s.Object{ + "bytes": c.Int("avg_size_in_bytes", s.Required), + }, + }, + "size": s.Object{ + "bytes": c.Int("total_size_in_bytes", s.Required), + }, + }, c.DictOptional), }), - "store": c.Dict("store", s.Schema{ - "size": s.Object{ - "bytes": c.Int("size_in_bytes"), - }, - "total_data_set_size": s.Object{ - "bytes": c.Int("total_data_set_size_in_bytes", s.Optional), - }, - }), - "segments": c.Dict("segments", s.Schema{ - "count": c.Int("count"), - "memory": s.Object{ - "bytes": c.Int("memory_in_bytes"), - }, - }), - "indexing": indexingDict, - "bulk": bulkStatsDict, - "search": searchDict, } -var indexingDict = c.Dict("indexing", s.Schema{ - "index": s.Object{ - "count": c.Int("index_total"), - "time": s.Object{ - "ms": c.Int("index_time_in_millis"), - }, - }, -}) - -var searchDict = c.Dict("search", s.Schema{ - "query": s.Object{ - "count": c.Int("query_total"), - "time": s.Object{ - "ms": c.Int("query_time_in_millis"), - }, - }, -}) - -var bulkStatsDict = c.Dict("bulk", s.Schema{ - "operations": s.Object{ - "count": c.Int("total_operations"), - }, - "time": s.Object{ - "avg": s.Object{ - "bytes": c.Int("avg_size_in_bytes"), - }, - }, - "size": s.Object{ - "bytes": c.Int("total_size_in_bytes"), - }, -}, c.DictOptional) - -func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXpack bool) error { - var all struct { - Data map[string]interface{} `json:"_all"` +type IndexSummaryMetricSet struct { + Primaries IndexSummary `json:"primaries"` + Total IndexSummary `json:"total"` +} + +type IndexSummary struct { + Docs DocsSection `json:"docs"` + Store StoreSection `json:"store"` + Indexing IndexingSection `json:"indexing"` + Search SearchSection `json:"search"` + Segments SegmentSection `json:"segments"` +} + +type DocsSection struct { + Count int64 `json:"count"` + Deleted int64 `json:"deleted"` +} + +type StoreSection struct { + Size struct { + Bytes int64 `json:"bytes"` + } `json:"size"` + TotalDataSetSize struct { + Bytes int64 `json:"bytes"` + } `json:"total_data_set_size"` +} + +type IndexingSection struct { + Index struct { + Count int64 `json:"count"` + Time struct { + Ms int64 `json:"ms"` + } `json:"time"` + } `json:"index"` +} + +type SearchSection struct { + Query struct { + Count int64 `json:"count"` + Time struct { + Ms int64 `json:"ms"` + } `json:"time"` + } `json:"query"` +} + +type SegmentSection struct { + Count int64 `json:"count"` + Memory struct { + Bytes int64 `json:"bytes"` + } `json:"memory"` +} + +func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXPack bool) error { + var wrapper nodeStatsWrapper + if err := json.Unmarshal(content, &wrapper); err != nil { + return fmt.Errorf("failure parsing NodeStats API response: %w", err) } - err := json.Unmarshal(content, &all) - if err != nil { - return fmt.Errorf("failure parsing Elasticsearch Stats API response: %w", err) + if len(wrapper.Nodes) == 0 { + return fmt.Errorf("no nodes found in NodeStats response") + } + + var total IndexSummary + for nodeKey, raw := range wrapper.Nodes { + err := addNodeMetrics(raw, &total) + if err != nil { + return fmt.Errorf("error processing node %q: %w", nodeKey, err) + } + } + + event := buildEvent(&info, &total, isXPack) + r.Event(event) + return nil +} + +func addNodeMetrics(rawNode interface{}, summary *IndexSummary) error { + nodeMap, ok := rawNode.(map[string]interface{}) + if !ok { + return fmt.Errorf("node is not a map") } - fields, err := schema.Apply(all.Data, s.FailOnRequired) + validated, err := nodeItemSchema.Apply(nodeMap, s.FailOnRequired) if err != nil { - return fmt.Errorf("failure applying stats schema: %w", err) + return err + } + + incrementValue := func(dst *int64, path ...string) { + if v, err := getInt64(validated, path...); err == nil { + *dst += v + } + } + + // Docs + incrementValue(&summary.Docs.Count, "indices", "docs", "count") + incrementValue(&summary.Docs.Deleted, "indices", "docs", "deleted") + + // Store + incrementValue(&summary.Store.Size.Bytes, "indices", "store", "size", "bytes") + incrementValue(&summary.Store.TotalDataSetSize.Bytes, "indices", "store", "total_data_set_size", "bytes") + + // Indexing + incrementValue(&summary.Indexing.Index.Count, "indices", "indexing", "index", "count") + incrementValue(&summary.Indexing.Index.Time.Ms, "indices", "indexing", "index", "time", "ms") + + // Search + incrementValue(&summary.Search.Query.Count, "indices", "search", "query", "count") + incrementValue(&summary.Search.Query.Time.Ms, "indices", "search", "query", "time", "ms") + + // Segments + incrementValue(&summary.Segments.Count, "indices", "segments", "count") + incrementValue(&summary.Segments.Memory.Bytes, "indices", "segments", "memory", "bytes") + + return nil +} + +func getInt64(m mapstr.M, path ...string) (int64, error) { + current := interface{}(m) + for _, key := range path { + mm, ok := current.(mapstr.M) + if !ok { + return 0, fmt.Errorf("expected mapstr.M at %q, got %T", key, current) + } + val, ok := mm[key] + if !ok { + return 0, fmt.Errorf("missing key: %q", key) + } + current = val + } + + i, ok := current.(int64) + if !ok { + return 0, fmt.Errorf("expected int64 at path %v, got %T", path, current) + } + return i, nil +} + +func buildEvent(info *elasticsearch.Info, summary *IndexSummary, isXPack bool) mb.Event { + eventNew := map[string]interface{}{ + "primaries": summary, + "total": summary, } var event mb.Event @@ -116,16 +236,12 @@ func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXp _, _ = event.ModuleFields.Put("cluster.name", info.ClusterName) _, _ = event.ModuleFields.Put("cluster.id", info.ClusterID) - event.MetricSetFields = fields + event.MetricSetFields = eventNew - // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` - // When using Agent, the index name is overwritten anyways. - if isXpack { + if isXPack { index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) event.Index = index } - r.Event(event) - - return nil + return event } diff --git a/metricbeat/module/elasticsearch/index_summary/data_test.go b/metricbeat/module/elasticsearch/index_summary/data_test.go index 3df0e61b2698..9e8502049924 100644 --- a/metricbeat/module/elasticsearch/index_summary/data_test.go +++ b/metricbeat/module/elasticsearch/index_summary/data_test.go @@ -20,9 +20,9 @@ package index_summary import ( - "io/ioutil" "net/http" "net/http/httptest" + "os" "testing" "github.com/stretchr/testify/require" @@ -48,7 +48,7 @@ func createEsMuxer(license string) *http.ServeMux { http.NotFound(w, r) } - input, _ := ioutil.ReadFile("../index/_meta/test/root.710.json") + input, _ := os.ReadFile("../index/_meta/test/root.710.json") w.Write(input) } licenseHandler := func(w http.ResponseWriter, r *http.Request) { @@ -62,7 +62,7 @@ func createEsMuxer(license string) *http.ServeMux { mux.Handle("/_xpack/license", http.HandlerFunc(licenseHandler)) // for before 7.0 mux.Handle("/", http.HandlerFunc(rootHandler)) mux.Handle("/_stats", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - content, _ := ioutil.ReadFile("../index/_meta/test/stats.700-alpha1.json") + content, _ := os.ReadFile("../index/_meta/test/stats.700-alpha1.json") w.Write(content) })) @@ -82,17 +82,124 @@ func TestData(t *testing.T) { } func TestMapper(t *testing.T) { - elasticsearch.TestMapperWithInfo(t, "../index/_meta/test/stats.*.json", eventMapping) + elasticsearch.TestMapperWithInfo(t, "_meta/test/node_stats_v*17.json", eventMapping) +} + +func TestSummaryFromNodeStatsWithExpectedEventsV817(t *testing.T) { + elasticsearch.TestMapperWithExpectedEvents( + t, + "_meta/test/node_stats_v817.json", + []string{ + "_meta/test/expected_event_8.17.json", + }, + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + true, + eventMapping, + ) +} + +func TestSummaryMissingField(t *testing.T) { + elasticsearch.TestMapperExpectingError( + t, + "_meta/test/node_stats_v817_missing_fields.json", + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + true, + "error processing node \"Hwq8Kg1eRNaFnFJKrKoqjA\": 1 error: key `indices.docs.count` not found", + eventMapping, + ) +} + +func TestSummaryMissingBlock(t *testing.T) { + elasticsearch.TestMapperExpectingError( + t, + "_meta/test/node_stats_v817_missing_block.json", + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + true, + "error processing node \"Hwq8Kg1eRNaFnFJKrKoqjA\": 1 error: key `indices.segments` not found", + eventMapping, + ) +} + +func TestSummaryWrongFieldType_String(t *testing.T) { + elasticsearch.TestMapperExpectingError( + t, + "_meta/test/node_stats_v717_field_as_string.json", + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + true, + "error processing node \"vF3ak-83RKu_020pnVZJ_w\": 1 error: wrong format in `indices.store.size_in_bytes`: expected integer, found string", + eventMapping, + ) +} + +func TestSummaryFromNodeStatsWithExpectedEventsV717(t *testing.T) { + elasticsearch.TestMapperWithExpectedEvents( + t, + "_meta/test/node_stats_v717.json", + []string{ + "_meta/test/expected_event_7.17.json", + }, + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + true, + eventMapping, + ) +} + +func TestSummaryFromNodeStatsWithExpectedEventsXPackV817(t *testing.T) { + elasticsearch.TestMapperWithExpectedEvents( + t, + "_meta/test/node_stats_v817.json", + []string{ + "_meta/test/expected_event_xpack_8.17.json", + }, + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + false, + eventMapping, + ) +} + +func TestSummaryFromNodeStatsWithExpectedEventsXPackV717(t *testing.T) { + elasticsearch.TestMapperWithExpectedEvents( + t, + "_meta/test/node_stats_v717.json", + []string{ + "_meta/test/expected_event_xpack_7.17.json", + }, + elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", + }, + false, + eventMapping, + ) } func TestEmpty(t *testing.T) { - input, err := ioutil.ReadFile("../index/_meta/test/empty.512.json") - require.NoError(t, err) + input, errReading := os.ReadFile("_meta/test/node_stats_empty.json") + require.NoError(t, errReading) reporter := &mbtest.CapturingReporterV2{} - eventMapping(reporter, info, input, true) - require.Empty(t, reporter.GetErrors()) - require.Equal(t, 1, len(reporter.GetEvents())) + err := eventMapping(reporter, info, input, true) + + require.ErrorContains(t, err, "no nodes found in NodeStats response") + require.Equal(t, 0, len(reporter.GetEvents())) } func getConfig(host string) map[string]interface{} { diff --git a/metricbeat/module/elasticsearch/index_summary/index_summary.go b/metricbeat/module/elasticsearch/index_summary/index_summary.go index 7a2c8bbcd7e9..105c179b1b17 100644 --- a/metricbeat/module/elasticsearch/index_summary/index_summary.go +++ b/metricbeat/module/elasticsearch/index_summary/index_summary.go @@ -24,8 +24,6 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" - - "github.com/elastic/elastic-agent-libs/version" ) // init registers the MetricSet with the central registry. @@ -38,10 +36,9 @@ func init() { } const ( - statsPath = "/_stats" + nodeStatsPath = "/_nodes/stats" - onlyClusterLevel = "level=cluster" - allowClosedIndices = "&forbid_closed_indices=false" + nodeStatsParameters = "level=node&filter_path=nodes.*.indices.docs,nodes.*.indices.indexing.index_total,nodes.*.indices.indexing.index_time_in_millis,nodes.*.indices.search.query_total,nodes.*.indices.search.query_time_in_millis,nodes.*.indices.segments.count,nodes.*.indices.segments.memory_in_bytes,nodes.*.indices.store.size_in_bytes,nodes.*.indices.store.total_data_set_size_in_bytes" ) var ( @@ -59,7 +56,7 @@ type MetricSet struct { // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Get the stats from the local node - ms, err := elasticsearch.NewMetricSet(base, statsPath) + ms, err := elasticsearch.NewMetricSet(base, nodeStatsPath) if err != nil { return nil, err } @@ -76,16 +73,16 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return nil } - info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+statsPath) + info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+nodeStatsPath) if err != nil { return fmt.Errorf("failed to get info from Elasticsearch: %w", err) } - if err := m.updateServicePath(*info.Version.Number); err != nil { + if err := m.updateServicePath(); err != nil { return err } - content, err := m.HTTP.FetchContent() + content, err := m.FetchContent() if err != nil { return err } @@ -93,8 +90,8 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return eventMapping(r, info, content, m.XPackEnabled) } -func (m *MetricSet) updateServicePath(esVersion version.V) error { - p, err := getServicePath(esVersion) +func (m *MetricSet) updateServicePath() error { + p, err := getServicePath() if err != nil { return err } @@ -103,17 +100,14 @@ func (m *MetricSet) updateServicePath(esVersion version.V) error { return nil } -func getServicePath(esVersion version.V) (string, error) { - currPath := statsPath +func getServicePath() (string, error) { + currPath := nodeStatsPath u, err := url.Parse(currPath) if err != nil { return "", err } - u.RawQuery += onlyClusterLevel - if !esVersion.LessThan(elasticsearch.BulkStatsAvailableVersion) { - u.RawQuery += allowClosedIndices - } + u.RawQuery += nodeStatsParameters return u.String(), nil } diff --git a/metricbeat/module/elasticsearch/index_summary/index_summary_test.go b/metricbeat/module/elasticsearch/index_summary/index_summary_test.go new file mode 100644 index 000000000000..e61d26db4304 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/index_summary_test.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !integration + +package index_summary + +import ( + "testing" +) + +func TestGetServicePath(t *testing.T) { + expectedPath := "/_nodes/stats?level=node&filter_path=nodes.*.indices.docs,nodes.*.indices.indexing.index_total,nodes.*.indices.indexing.index_time_in_millis,nodes.*.indices.search.query_total,nodes.*.indices.search.query_time_in_millis,nodes.*.indices.segments.count,nodes.*.indices.segments.memory_in_bytes,nodes.*.indices.store.size_in_bytes,nodes.*.indices.store.total_data_set_size_in_bytes" + path, err := getServicePath() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if path != expectedPath { + t.Errorf("expected path %q, got %q", expectedPath, path) + } +} diff --git a/metricbeat/module/elasticsearch/testing.go b/metricbeat/module/elasticsearch/testing.go index 63f041c44fd5..b0fe6748c461 100644 --- a/metricbeat/module/elasticsearch/testing.go +++ b/metricbeat/module/elasticsearch/testing.go @@ -20,10 +20,14 @@ package elasticsearch import ( - "io/ioutil" + "encoding/json" + "fmt" + "os" "path/filepath" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/metricbeat/helper" @@ -41,7 +45,7 @@ func TestMapper(t *testing.T, glob string, mapper func(mb.ReporterV2, []byte) er for _, f := range files { t.Run(f, func(t *testing.T) { - input, err := ioutil.ReadFile(f) + input, err := os.ReadFile(f) require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} @@ -58,7 +62,7 @@ func TestMapperWithInfo(t *testing.T, glob string, mapper func(mb.ReporterV2, In files, err := filepath.Glob(glob) require.NoError(t, err) // Makes sure glob matches at least 1 file - require.True(t, len(files) > 0) + require.True(t, len(files) > 0, "Glob should match at least one file") info := Info{ ClusterID: "1234", @@ -67,18 +71,85 @@ func TestMapperWithInfo(t *testing.T, glob string, mapper func(mb.ReporterV2, In for _, f := range files { t.Run(f, func(t *testing.T) { - input, err := ioutil.ReadFile(f) + input, err := os.ReadFile(f) require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} err = mapper(reporter, info, input, true) require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) require.Equal(t, 0, len(reporter.GetErrors())) }) } } +func TestMapperWithExpectedEvents( + t *testing.T, + inputPath string, + expectedFiles []string, + info Info, + isXPack bool, + mapper func(mb.ReporterV2, Info, []byte, bool) error, +) { + input, err := os.ReadFile(inputPath) + require.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + err = mapper(reporter, info, input, isXPack) + require.NoError(t, err) + + events := reporter.GetEvents() + + expected := loadExpectedEventsFromFiles(t, expectedFiles) + require.Equal(t, len(expected), len(events), "Number of events mismatch") + + for i, ev := range events { + actualBytes, err := json.Marshal(ev) + require.NoError(t, err) + + var actual map[string]interface{} + err = json.Unmarshal(actualBytes, &actual) + require.NoError(t, err) + + assert.Equal(t, expected[i], actual, fmt.Sprintf("Mismatch in event #%d", i)) + } +} + +func TestMapperExpectingError( + t *testing.T, + inputPath string, + info Info, + isXPack bool, + errorMessage string, + mapper func(mb.ReporterV2, Info, []byte, bool) error, +) { + input, err := os.ReadFile(inputPath) + require.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + err = mapper(reporter, info, input, isXPack) + require.ErrorContains(t, err, errorMessage) + + events := reporter.GetEvents() + require.Equal(t, 0, len(events), "Number of events mismatch") +} + +func loadExpectedEventsFromFiles(t *testing.T, files []string) []map[string]interface{} { + expected := make([]map[string]interface{}, 0, len(files)) + for _, f := range files { + content, err := os.ReadFile(f) + require.NoError(t, err) + + var ev map[string]interface{} + err = json.Unmarshal(content, &ev) + require.NoError(t, err) + + expected = append(expected, ev) + } + return expected +} + // TestMapperWithMetricSetAndInfo tests mapping methods with Info fields func TestMapperWithMetricSetAndInfo(t *testing.T, glob string, ms MetricSetAPI, mapper func(mb.ReporterV2, MetricSetAPI, Info, []byte, bool) error) { files, err := filepath.Glob(glob) @@ -93,7 +164,7 @@ func TestMapperWithMetricSetAndInfo(t *testing.T, glob string, ms MetricSetAPI, for _, f := range files { t.Run(f, func(t *testing.T) { - input, err := ioutil.ReadFile(f) + input, err := os.ReadFile(f) require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} @@ -125,7 +196,7 @@ func TestMapperWithHttpHelper(t *testing.T, glob string, httpClient *helper.HTTP for _, f := range files { t.Run(f, func(t *testing.T) { - input, err := ioutil.ReadFile(f) + input, err := os.ReadFile(f) require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{}