Skip to content

Commit

Permalink
Add OS mem stats to node-stats telemetry device (#1530)
Browse files Browse the repository at this point in the history 
With this commit we add the OS memory stats to default list of
collected node-stats metrics.
  • Loading branch information
@b-deam
b-deam authored Jun 28, 2022
1 parent b7159de commit 591f612
Show file tree
Hide file tree
Showing 3 changed files with 99 additions and 1 deletion.
3 changes: 2 additions & 1 deletion docs/telemetry.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ The node-stats telemetry device regularly calls the `cluster node-stats API <htt
* Thread pool stats (key ``thread_pool`` in the node-stats API)
* JVM buffer pool stats (key ``jvm.buffer_pools`` in the node-stats API)
* JVM gc stats (key ``jvm.gc`` in the node-stats API)
* OS mem stats (key ``os.mem`` in the node-stats API)
* JVM mem stats (key ``jvm.mem`` in the node-stats API)
* Circuit breaker stats (key ``breakers`` in the node-stats API)
* Network-related stats (key ``transport`` in the node-stats API)
Expand All @@ -128,7 +129,7 @@ Supported telemetry parameters:
* ``node-stats-include-buffer-pools`` (default: ``true``): A boolean indicating whether buffer pool stats should be included.
* ``node-stats-include-breakers`` (default: ``true``): A boolean indicating whether circuit breaker stats should be included.
* ``node-stats-include-gc`` (default: ``true``): A boolean indicating whether JVM gc stats should be included.
* ``node-stats-include-mem`` (default: ``true``): A boolean indicating whether JVM heap stats should be included.
* ``node-stats-include-mem`` (default: ``true``): A boolean indicating whether both JVM heap, and OS mem stats should be included.
* ``node-stats-include-network`` (default: ``true``): A boolean indicating whether network-related stats should be included.
* ``node-stats-include-process`` (default: ``true``): A boolean indicating whether process cpu stats should be included.
* ``node-stats-include-indexing-pressure`` (default: ``true``): A boolean indicating whether indexing pressuer stats should be included.
Expand Down
4 changes: 4 additions & 0 deletions esrally/telemetry.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -837,6 +837,7 @@ def record(self):
collected_node_stats.update(self.jvm_buffer_pool_stats(node_name, node_stats))
if self.include_mem_stats:
collected_node_stats.update(self.jvm_mem_stats(node_name, node_stats))
collected_node_stats.update(self.os_mem_stats(node_name, node_stats))
if self.include_gc_stats:
collected_node_stats.update(self.jvm_gc_stats(node_name, node_stats))
if self.include_network:
Expand Down Expand Up @@ -896,6 +897,9 @@ def jvm_buffer_pool_stats(self, node_name, node_stats):
def jvm_mem_stats(self, node_name, node_stats):
return self.flatten_stats_fields(prefix="jvm_mem", stats=node_stats["jvm"]["mem"])

def os_mem_stats(self, node_name, node_stats):
return self.flatten_stats_fields(prefix="os_mem", stats=node_stats["os"]["mem"])

def jvm_gc_stats(self, node_name, node_stats):
return self.flatten_stats_fields(prefix="jvm_gc", stats=node_stats["jvm"]["gc"])

Expand Down
93 changes: 93 additions & 0 deletions tests/telemetry_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1918,6 +1918,32 @@ class TestNodeStatsRecorder:
}
},
},
"os": {
"timestamp": 1655950949872,
"cpu": {"percent": 3, "load_average": {"1m": 3.38, "5m": 3.79, "15m": 3.84}},
"mem": {
"total_in_bytes": 62277025792,
"free_in_bytes": 4934840320,
"used_in_bytes": 57342185472,
"free_percent": 8,
"used_percent": 92,
},
"swap": {"total_in_bytes": 0, "free_in_bytes": 0, "used_in_bytes": 0},
"cgroup": {
"cpuacct": {"control_group": "/", "usage_nanos": 1394207523870751},
"cpu": {
"control_group": "/",
"cfs_period_micros": 100000,
"cfs_quota_micros": 793162,
"stat": {
"number_of_elapsed_periods": 41092415,
"number_of_times_throttled": 41890,
"time_throttled_nanos": 29380593023188,
},
},
"memory": {"control_group": "/", "limit_in_bytes": "62277025792", "usage_in_bytes": "57342185472"},
},
},
"process": {
"timestamp": 1526045135857,
"open_file_descriptors": 312,
Expand Down Expand Up @@ -2045,6 +2071,11 @@ class TestNodeStatsRecorder:
"jvm_gc_collectors_young_collection_time_in_millis": 309,
"jvm_gc_collectors_old_collection_count": 2,
"jvm_gc_collectors_old_collection_time_in_millis": 229,
"os_mem_total_in_bytes": 62277025792,
"os_mem_free_in_bytes": 4934840320,
"os_mem_used_in_bytes": 57342185472,
"os_mem_free_percent": 8,
"os_mem_used_percent": 92,
"process_cpu_percent": 10,
"process_cpu_total_in_millis": 56520,
"breakers_parent_limit_size_in_bytes": 726571417,
Expand Down Expand Up @@ -2270,6 +2301,32 @@ def test_stores_all_nodes_stats(self, metrics_store_put_doc):
"total_virtual_in_bytes": 2472173568,
},
},
"os": {
"timestamp": 1655950949872,
"cpu": {"percent": 3, "load_average": {"1m": 3.38, "5m": 3.79, "15m": 3.84}},
"mem": {
"total_in_bytes": 62277025792,
"free_in_bytes": 4934840320,
"used_in_bytes": 57342185472,
"free_percent": 8,
"used_percent": 92,
},
"swap": {"total_in_bytes": 0, "free_in_bytes": 0, "used_in_bytes": 0},
"cgroup": {
"cpuacct": {"control_group": "/", "usage_nanos": 1394207523870751},
"cpu": {
"control_group": "/",
"cfs_period_micros": 100000,
"cfs_quota_micros": 793162,
"stat": {
"number_of_elapsed_periods": 41092415,
"number_of_times_throttled": 41890,
"time_throttled_nanos": 29380593023188,
},
},
"memory": {"control_group": "/", "limit_in_bytes": "62277025792", "usage_in_bytes": "57342185472"},
},
},
"thread_pool": {
"generic": {
"threads": 4,
Expand Down Expand Up @@ -2402,6 +2459,11 @@ def test_stores_all_nodes_stats(self, metrics_store_put_doc):
"jvm_gc_collectors_young_collection_time_in_millis": 309,
"jvm_gc_collectors_old_collection_count": 2,
"jvm_gc_collectors_old_collection_time_in_millis": 229,
"os_mem_total_in_bytes": 62277025792,
"os_mem_free_in_bytes": 4934840320,
"os_mem_used_in_bytes": 57342185472,
"os_mem_free_percent": 8,
"os_mem_used_percent": 92,
"transport_rx_count": 77,
"transport_rx_size_in_bytes": 98723498,
"transport_server_open": 12,
Expand Down Expand Up @@ -2568,6 +2630,32 @@ def test_stores_selected_indices_metrics_from_nodes_stats(self, metrics_store_pu
}
},
},
"os": {
"timestamp": 1655950949872,
"cpu": {"percent": 3, "load_average": {"1m": 3.38, "5m": 3.79, "15m": 3.84}},
"mem": {
"total_in_bytes": 62277025792,
"free_in_bytes": 4934840320,
"used_in_bytes": 57342185472,
"free_percent": 8,
"used_percent": 92,
},
"swap": {"total_in_bytes": 0, "free_in_bytes": 0, "used_in_bytes": 0},
"cgroup": {
"cpuacct": {"control_group": "/", "usage_nanos": 1394207523870751},
"cpu": {
"control_group": "/",
"cfs_period_micros": 100000,
"cfs_quota_micros": 793162,
"stat": {
"number_of_elapsed_periods": 41092415,
"number_of_times_throttled": 41890,
"time_throttled_nanos": 29380593023188,
},
},
"memory": {"control_group": "/", "limit_in_bytes": "62277025792", "usage_in_bytes": "57342185472"},
},
},
"process": {
"timestamp": 1526045135857,
"open_file_descriptors": 312,
Expand Down Expand Up @@ -2682,6 +2770,11 @@ def test_stores_selected_indices_metrics_from_nodes_stats(self, metrics_store_pu
"jvm_gc_collectors_young_collection_time_in_millis": 309,
"jvm_gc_collectors_old_collection_count": 2,
"jvm_gc_collectors_old_collection_time_in_millis": 229,
"os_mem_total_in_bytes": 62277025792,
"os_mem_free_in_bytes": 4934840320,
"os_mem_used_in_bytes": 57342185472,
"os_mem_free_percent": 8,
"os_mem_used_percent": 92,
"transport_rx_count": 77,
"transport_rx_size_in_bytes": 98723498,
"transport_server_open": 12,
Expand Down

0 comments on commit 591f612

Please sign in to comment.