diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index a223eaf8c377..f97f80d9ba6a 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -23,6 +23,7 @@ type NodeInfo struct { TransportAddress string `json:"transport_address"` IP string `json:"ip"` Name string `json:"name"` + ID string } // GetClusterID fetches cluster id for given nodeID @@ -147,7 +148,9 @@ func GetNodeInfo(http *helper.HTTP, uri string, nodeID string) (*NodeInfo, error // _local will only fetch one node info. First entry is node name for k, v := range nodesStruct.Nodes { - if k == nodeID { + // In case the nodeID is empty, first node info will be returned + if k == nodeID || nodeID == "" { + v.ID = k return v, nil } } diff --git a/metricbeat/module/elasticsearch/index/_meta/test/stats.700-alpha1.json b/metricbeat/module/elasticsearch/index/_meta/test/stats.700-alpha1.json new file mode 100644 index 000000000000..d50533f138e6 --- /dev/null +++ b/metricbeat/module/elasticsearch/index/_meta/test/stats.700-alpha1.json @@ -0,0 +1,1473 @@ +{ + "_shards" : { + "total" : 6, + "successful" : 5, + "failed" : 0 + }, + "_all" : { + "primaries" : { + "docs" : { + "count" : 41, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 174153 + }, + "indexing" : { + "index_total" : 66, + "index_time_in_millis" : 2239, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 6, + "time_in_millis" : 17, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 6, + "missing_time_in_millis" : 17, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 3, + "query_time_in_millis" : 674, + "query_current" : 0, + "fetch_total" : 1, + "fetch_time_in_millis" : 23, + "fetch_current" : 0, + "scroll_total" : 1, + "scroll_time_in_millis" : 1068, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 104857600 + }, + "refresh" : { + "total" : 26, + "total_time_in_millis" : 3076, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 17, + "total_time_in_millis" : 65 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 14, + "memory_in_bytes" : 60804, + "terms_memory_in_bytes" : 51071, + "stored_fields_memory_in_bytes" : 4368, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 2304, + "points_memory_in_bytes" : 861, + "doc_values_memory_in_bytes" : 2200, + "index_writer_memory_in_bytes" : 4456672, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 288, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 66, + "size_in_bytes" : 87028, + "uncommitted_operations" : 66, + "uncommitted_size_in_bytes" : 87028, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 2 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + }, + "total" : { + "docs" : { + "count" : 41, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 174153 + }, + "indexing" : { + "index_total" : 66, + "index_time_in_millis" : 2239, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 6, + "time_in_millis" : 17, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 6, + "missing_time_in_millis" : 17, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 3, + "query_time_in_millis" : 674, + "query_current" : 0, + "fetch_total" : 1, + "fetch_time_in_millis" : 23, + "fetch_current" : 0, + "scroll_total" : 1, + "scroll_time_in_millis" : 1068, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 104857600 + }, + "refresh" : { + "total" : 26, + "total_time_in_millis" : 3076, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 17, + "total_time_in_millis" : 65 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 14, + "memory_in_bytes" : 60804, + "terms_memory_in_bytes" : 51071, + "stored_fields_memory_in_bytes" : 4368, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 2304, + "points_memory_in_bytes" : 861, + "doc_values_memory_in_bytes" : 2200, + "index_writer_memory_in_bytes" : 4456672, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 288, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 66, + "size_in_bytes" : 87028, + "uncommitted_operations" : 66, + "uncommitted_size_in_bytes" : 87028, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 2 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + } + }, + "indices" : { + ".monitoring-kibana-6-2018.05.18" : { + "primaries" : { + "docs" : { + "count" : 3, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 29114 + }, + "indexing" : { + "index_total" : 3, + "index_time_in_millis" : 449, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 0, + "time_in_millis" : 0, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 0, + "missing_time_in_millis" : 0, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 1, + "query_time_in_millis" : 135, + "query_current" : 0, + "fetch_total" : 0, + "fetch_time_in_millis" : 0, + "fetch_current" : 0, + "scroll_total" : 0, + "scroll_time_in_millis" : 0, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 20971520 + }, + "refresh" : { + "total" : 5, + "total_time_in_millis" : 553, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 4, + "total_time_in_millis" : 13 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 3, + "memory_in_bytes" : 11586, + "terms_memory_in_bytes" : 10317, + "stored_fields_memory_in_bytes" : 936, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 0, + "points_memory_in_bytes" : 129, + "doc_values_memory_in_bytes" : 204, + "index_writer_memory_in_bytes" : 0, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 0, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 3, + "size_in_bytes" : 3814, + "uncommitted_operations" : 3, + "uncommitted_size_in_bytes" : 3814, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 1 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + }, + "total" : { + "docs" : { + "count" : 3, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 29114 + }, + "indexing" : { + "index_total" : 3, + "index_time_in_millis" : 449, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 0, + "time_in_millis" : 0, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 0, + "missing_time_in_millis" : 0, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 1, + "query_time_in_millis" : 135, + "query_current" : 0, + "fetch_total" : 0, + "fetch_time_in_millis" : 0, + "fetch_current" : 0, + "scroll_total" : 0, + "scroll_time_in_millis" : 0, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 20971520 + }, + "refresh" : { + "total" : 5, + "total_time_in_millis" : 553, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 4, + "total_time_in_millis" : 13 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 3, + "memory_in_bytes" : 11586, + "terms_memory_in_bytes" : 10317, + "stored_fields_memory_in_bytes" : 936, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 0, + "points_memory_in_bytes" : 129, + "doc_values_memory_in_bytes" : 204, + "index_writer_memory_in_bytes" : 0, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 0, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 3, + "size_in_bytes" : 3814, + "uncommitted_operations" : 3, + "uncommitted_size_in_bytes" : 3814, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 1 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + } + }, + ".monitoring-logstash-6-2018.05.18" : { + "primaries" : { + "docs" : { + "count" : 12, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 28695 + }, + "indexing" : { + "index_total" : 4, + "index_time_in_millis" : 46, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 0, + "time_in_millis" : 0, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 0, + "missing_time_in_millis" : 0, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 0, + "query_time_in_millis" : 0, + "query_current" : 0, + "fetch_total" : 0, + "fetch_time_in_millis" : 0, + "fetch_current" : 0, + "scroll_total" : 0, + "scroll_time_in_millis" : 0, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 20971520 + }, + "refresh" : { + "total" : 4, + "total_time_in_millis" : 233, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 3, + "total_time_in_millis" : 36 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 2, + "memory_in_bytes" : 12760, + "terms_memory_in_bytes" : 11828, + "stored_fields_memory_in_bytes" : 624, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 0, + "points_memory_in_bytes" : 172, + "doc_values_memory_in_bytes" : 136, + "index_writer_memory_in_bytes" : 0, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 288, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 4, + "size_in_bytes" : 12826, + "uncommitted_operations" : 4, + "uncommitted_size_in_bytes" : 12826, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 0 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + }, + "total" : { + "docs" : { + "count" : 12, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 28695 + }, + "indexing" : { + "index_total" : 4, + "index_time_in_millis" : 46, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 0, + "time_in_millis" : 0, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 0, + "missing_time_in_millis" : 0, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 0, + "query_time_in_millis" : 0, + "query_current" : 0, + "fetch_total" : 0, + "fetch_time_in_millis" : 0, + "fetch_current" : 0, + "scroll_total" : 0, + "scroll_time_in_millis" : 0, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 20971520 + }, + "refresh" : { + "total" : 4, + "total_time_in_millis" : 233, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 3, + "total_time_in_millis" : 36 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 2, + "memory_in_bytes" : 12760, + "terms_memory_in_bytes" : 11828, + "stored_fields_memory_in_bytes" : 624, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 0, + "points_memory_in_bytes" : 172, + "doc_values_memory_in_bytes" : 136, + "index_writer_memory_in_bytes" : 0, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 288, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 4, + "size_in_bytes" : 12826, + "uncommitted_operations" : 4, + "uncommitted_size_in_bytes" : 12826, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 0 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + } + }, + ".monitoring-es-6-2018.05.18" : { + "primaries" : { + "docs" : { + "count" : 20, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 56958 + }, + "indexing" : { + "index_total" : 20, + "index_time_in_millis" : 375, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 0, + "time_in_millis" : 0, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 0, + "missing_time_in_millis" : 0, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 0, + "query_time_in_millis" : 0, + "query_current" : 0, + "fetch_total" : 0, + "fetch_time_in_millis" : 0, + "fetch_current" : 0, + "scroll_total" : 0, + "scroll_time_in_millis" : 0, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 20971520 + }, + "refresh" : { + "total" : 5, + "total_time_in_millis" : 1053, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 3, + "total_time_in_millis" : 0 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 3, + "memory_in_bytes" : 13730, + "terms_memory_in_bytes" : 10992, + "stored_fields_memory_in_bytes" : 936, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 0, + "points_memory_in_bytes" : 534, + "doc_values_memory_in_bytes" : 1268, + "index_writer_memory_in_bytes" : 0, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 0, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 20, + "size_in_bytes" : 25320, + "uncommitted_operations" : 20, + "uncommitted_size_in_bytes" : 25320, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 0 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + }, + "total" : { + "docs" : { + "count" : 20, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 56958 + }, + "indexing" : { + "index_total" : 20, + "index_time_in_millis" : 375, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 0, + "time_in_millis" : 0, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 0, + "missing_time_in_millis" : 0, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 0, + "query_time_in_millis" : 0, + "query_current" : 0, + "fetch_total" : 0, + "fetch_time_in_millis" : 0, + "fetch_current" : 0, + "scroll_total" : 0, + "scroll_time_in_millis" : 0, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 20971520 + }, + "refresh" : { + "total" : 5, + "total_time_in_millis" : 1053, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 3, + "total_time_in_millis" : 0 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 3, + "memory_in_bytes" : 13730, + "terms_memory_in_bytes" : 10992, + "stored_fields_memory_in_bytes" : 936, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 0, + "points_memory_in_bytes" : 534, + "doc_values_memory_in_bytes" : 1268, + "index_writer_memory_in_bytes" : 0, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 0, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 20, + "size_in_bytes" : 25320, + "uncommitted_operations" : 20, + "uncommitted_size_in_bytes" : 25320, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 0 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + } + }, + "metricbeat-7.0.0-alpha1-2018.05.18" : { + "primaries" : { + "docs" : { + "count" : 0, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 230 + }, + "indexing" : { + "index_total" : 33, + "index_time_in_millis" : 328, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 0, + "time_in_millis" : 0, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 0, + "missing_time_in_millis" : 0, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 0, + "query_time_in_millis" : 0, + "query_current" : 0, + "fetch_total" : 0, + "fetch_time_in_millis" : 0, + "fetch_current" : 0, + "scroll_total" : 0, + "scroll_time_in_millis" : 0, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 20971520 + }, + "refresh" : { + "total" : 2, + "total_time_in_millis" : 0, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 1, + "total_time_in_millis" : 2 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 0, + "memory_in_bytes" : 0, + "terms_memory_in_bytes" : 0, + "stored_fields_memory_in_bytes" : 0, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 0, + "points_memory_in_bytes" : 0, + "doc_values_memory_in_bytes" : 0, + "index_writer_memory_in_bytes" : 4456672, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 0, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 33, + "size_in_bytes" : 18965, + "uncommitted_operations" : 33, + "uncommitted_size_in_bytes" : 18965, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 0 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + }, + "total" : { + "docs" : { + "count" : 0, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 230 + }, + "indexing" : { + "index_total" : 33, + "index_time_in_millis" : 328, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 0, + "time_in_millis" : 0, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 0, + "missing_time_in_millis" : 0, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 0, + "query_time_in_millis" : 0, + "query_current" : 0, + "fetch_total" : 0, + "fetch_time_in_millis" : 0, + "fetch_current" : 0, + "scroll_total" : 0, + "scroll_time_in_millis" : 0, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 20971520 + }, + "refresh" : { + "total" : 2, + "total_time_in_millis" : 0, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 1, + "total_time_in_millis" : 2 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 0, + "memory_in_bytes" : 0, + "terms_memory_in_bytes" : 0, + "stored_fields_memory_in_bytes" : 0, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 0, + "points_memory_in_bytes" : 0, + "doc_values_memory_in_bytes" : 0, + "index_writer_memory_in_bytes" : 4456672, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 0, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 33, + "size_in_bytes" : 18965, + "uncommitted_operations" : 33, + "uncommitted_size_in_bytes" : 18965, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 0 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + } + }, + ".watches" : { + "primaries" : { + "docs" : { + "count" : 6, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 59156 + }, + "indexing" : { + "index_total" : 6, + "index_time_in_millis" : 1041, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 6, + "time_in_millis" : 17, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 6, + "missing_time_in_millis" : 17, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 2, + "query_time_in_millis" : 539, + "query_current" : 0, + "fetch_total" : 1, + "fetch_time_in_millis" : 23, + "fetch_current" : 0, + "scroll_total" : 1, + "scroll_time_in_millis" : 1068, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 20971520 + }, + "refresh" : { + "total" : 10, + "total_time_in_millis" : 1237, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 6, + "total_time_in_millis" : 14 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 6, + "memory_in_bytes" : 22728, + "terms_memory_in_bytes" : 17934, + "stored_fields_memory_in_bytes" : 1872, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 2304, + "points_memory_in_bytes" : 26, + "doc_values_memory_in_bytes" : 592, + "index_writer_memory_in_bytes" : 0, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 0, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 6, + "size_in_bytes" : 26103, + "uncommitted_operations" : 6, + "uncommitted_size_in_bytes" : 26103, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 1 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + }, + "total" : { + "docs" : { + "count" : 6, + "deleted" : 0 + }, + "store" : { + "size_in_bytes" : 59156 + }, + "indexing" : { + "index_total" : 6, + "index_time_in_millis" : 1041, + "index_current" : 0, + "index_failed" : 0, + "delete_total" : 0, + "delete_time_in_millis" : 0, + "delete_current" : 0, + "noop_update_total" : 0, + "is_throttled" : false, + "throttle_time_in_millis" : 0 + }, + "get" : { + "total" : 6, + "time_in_millis" : 17, + "exists_total" : 0, + "exists_time_in_millis" : 0, + "missing_total" : 6, + "missing_time_in_millis" : 17, + "current" : 0 + }, + "search" : { + "open_contexts" : 0, + "query_total" : 2, + "query_time_in_millis" : 539, + "query_current" : 0, + "fetch_total" : 1, + "fetch_time_in_millis" : 23, + "fetch_current" : 0, + "scroll_total" : 1, + "scroll_time_in_millis" : 1068, + "scroll_current" : 0, + "suggest_total" : 0, + "suggest_time_in_millis" : 0, + "suggest_current" : 0 + }, + "merges" : { + "current" : 0, + "current_docs" : 0, + "current_size_in_bytes" : 0, + "total" : 0, + "total_time_in_millis" : 0, + "total_docs" : 0, + "total_size_in_bytes" : 0, + "total_stopped_time_in_millis" : 0, + "total_throttled_time_in_millis" : 0, + "total_auto_throttle_in_bytes" : 20971520 + }, + "refresh" : { + "total" : 10, + "total_time_in_millis" : 1237, + "listeners" : 0 + }, + "flush" : { + "total" : 0, + "periodic" : 0, + "total_time_in_millis" : 0 + }, + "warmer" : { + "current" : 0, + "total" : 6, + "total_time_in_millis" : 14 + }, + "query_cache" : { + "memory_size_in_bytes" : 0, + "total_count" : 0, + "hit_count" : 0, + "miss_count" : 0, + "cache_size" : 0, + "cache_count" : 0, + "evictions" : 0 + }, + "fielddata" : { + "memory_size_in_bytes" : 0, + "evictions" : 0 + }, + "completion" : { + "size_in_bytes" : 0 + }, + "segments" : { + "count" : 6, + "memory_in_bytes" : 22728, + "terms_memory_in_bytes" : 17934, + "stored_fields_memory_in_bytes" : 1872, + "term_vectors_memory_in_bytes" : 0, + "norms_memory_in_bytes" : 2304, + "points_memory_in_bytes" : 26, + "doc_values_memory_in_bytes" : 592, + "index_writer_memory_in_bytes" : 0, + "version_map_memory_in_bytes" : 0, + "fixed_bit_set_memory_in_bytes" : 0, + "max_unsafe_auto_id_timestamp" : -1, + "file_sizes" : { } + }, + "translog" : { + "operations" : 6, + "size_in_bytes" : 26103, + "uncommitted_operations" : 6, + "uncommitted_size_in_bytes" : 26103, + "earliest_last_modified_age" : 0 + }, + "request_cache" : { + "memory_size_in_bytes" : 0, + "evictions" : 0, + "hit_count" : 0, + "miss_count" : 1 + }, + "recovery" : { + "current_as_source" : 0, + "current_as_target" : 0, + "throttle_time_in_millis" : 0 + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/data-xpack.json b/metricbeat/module/elasticsearch/index_summary/_meta/data-xpack.json new file mode 100644 index 000000000000..ad68a4b786e7 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/_meta/data-xpack.json @@ -0,0 +1,60 @@ +{ + "_index": ".monitoring-es-6-2018.05.15", + "_type": "doc", + "_id": "CGdxY2MBLoB8ROglGWFD", + "_score": 1, + "_source": { + "cluster_uuid": "SFQoPJkbSc2-n_so4ZbUwA", + "timestamp": "2018-05-15T10:55:46.476Z", + "interval_ms": 10000, + "type": "indices_stats", + "source_node": { + "uuid": "YYSPA7QWSQyBQSYgIdOAKw", + "host": "127.0.0.1", + "transport_address": "127.0.0.1:9300", + "ip": "127.0.0.1", + "name": "YYSPA7Q", + "timestamp": "2018-05-15T10:55:46.360Z" + }, + "indices_stats": { + "_all": { + "primaries": { + "docs": { + "count": 509 + }, + "store": { + "size_in_bytes": 1377439 + }, + "indexing": { + "index_total": 531, + "index_time_in_millis": 5220, + "is_throttled": false, + "throttle_time_in_millis": 0 + }, + "search": { + "query_total": 908, + "query_time_in_millis": 10160 + } + }, + "total": { + "docs": { + "count": 509 + }, + "store": { + "size_in_bytes": 1377439 + }, + "indexing": { + "index_total": 531, + "index_time_in_millis": 5220, + "is_throttled": false, + "throttle_time_in_millis": 0 + }, + "search": { + "query_total": 908, + "query_time_in_millis": 10160 + } + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/index_summary/data_xpack.go b/metricbeat/module/elasticsearch/index_summary/data_xpack.go new file mode 100644 index 000000000000..bc20be21aab9 --- /dev/null +++ b/metricbeat/module/elasticsearch/index_summary/data_xpack.go @@ -0,0 +1,96 @@ +package index_summary + +import ( + "encoding/json" + "time" + + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/elasticsearch" +) + +var ( + schemaXPack = s.Schema{ + "primaries": c.Dict("primaries", s.Schema{ + "docs": c.Dict("docs", s.Schema{ + "count": c.Int("count"), + }), + "store": c.Dict("store", s.Schema{ + "size_in_bytes": c.Int("size_in_bytes"), + }), + "indexing": c.Dict("indexing", s.Schema{ + "index_total": c.Int("index_total"), + "index_time_in_millis": c.Int("index_time_in_millis"), + "is_throttled": c.Bool("is_throttled"), + "throttle_time_in_millis": c.Int("throttle_time_in_millis"), + }), + "search": c.Dict("search", s.Schema{ + "query_total": c.Int("query_total"), + "query_time_in_millis": c.Int("query_time_in_millis"), + }), + }), + "total": c.Dict("total", s.Schema{ + "docs": c.Dict("docs", s.Schema{ + "count": c.Int("count"), + }), + "store": c.Dict("store", s.Schema{ + "size_in_bytes": c.Int("size_in_bytes"), + }), + "indexing": c.Dict("indexing", s.Schema{ + "index_total": c.Int("index_total"), + "index_time_in_millis": c.Int("index_time_in_millis"), + "is_throttled": c.Bool("is_throttled"), + "throttle_time_in_millis": c.Int("throttle_time_in_millis"), + }), + "search": c.Dict("search", s.Schema{ + "query_total": c.Int("query_total"), + "query_time_in_millis": c.Int("query_time_in_millis"), + }), + }), + } +) + +func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) []error { + var all struct { + Data map[string]interface{} `json:"_all"` + } + + err := json.Unmarshal(content, &all) + if err != nil { + r.Error(err) + return []error{err} + } + + var errs []error + + fields, err := schemaXPack.Apply(all.Data) + if err != nil { + errs = append(errs, err) + } + + nodeInfo, err := elasticsearch.GetNodeInfo(m.HTTP, m.HostData().SanitizedURI+statsPath, "") + sourceNode := common.MapStr{ + "uuid": nodeInfo.ID, + "host": nodeInfo.Host, + "transport_address": nodeInfo.TransportAddress, + "ip": nodeInfo.IP, + "name": nodeInfo.Name, + "timestamp": common.Time(time.Now()), + } + event := mb.Event{} + event.RootFields = common.MapStr{} + event.RootFields.Put("indices_stats._all", fields) + event.RootFields.Put("cluser_uuid", info.ClusterID) + event.RootFields.Put("timestamp", common.Time(time.Now())) + event.RootFields.Put("interval_ms", m.Module().Config().Period/time.Millisecond) + event.RootFields.Put("type", "indices_stats") + event.RootFields.Put("source_node", sourceNode) + + event.Index = ".monitoring-es-6-mb" + + r.Event(event) + + return errs +} diff --git a/metricbeat/module/elasticsearch/index_summary/index_summary.go b/metricbeat/module/elasticsearch/index_summary/index_summary.go index 7594af70c7f2..83f2f345cbaf 100644 --- a/metricbeat/module/elasticsearch/index_summary/index_summary.go +++ b/metricbeat/module/elasticsearch/index_summary/index_summary.go @@ -3,7 +3,6 @@ package index_summary import ( "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" "github.com/elastic/beats/metricbeat/module/elasticsearch" @@ -18,38 +17,37 @@ func init() { ) } +const ( + statsPath = "/_stats" +) + var ( hostParser = parse.URLHostParserBuilder{ DefaultScheme: "http", PathConfigKey: "path", - DefaultPath: "_stats", }.Build() ) // MetricSet type defines all fields of the MetricSet type MetricSet struct { - mb.BaseMetricSet - http *helper.HTTP + *elasticsearch.MetricSet } // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Experimental("The elasticsearch index_summary metricset is experimental") - http, err := helper.NewHTTP(base) + // Get the stats from the local node + ms, err := elasticsearch.NewMetricSet(base, statsPath) if err != nil { return nil, err } - - return &MetricSet{ - base, - http, - }, nil + return &MetricSet{MetricSet: ms}, nil } // Fetch gathers stats for each index from the _stats API func (m *MetricSet) Fetch(r mb.ReporterV2) { - isMaster, err := elasticsearch.IsMaster(m.http, m.HostData().SanitizedURI) + isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+statsPath) if err != nil { r.Error(err) return @@ -61,17 +59,22 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } - content, err := m.http.FetchContent() + content, err := m.HTTP.FetchContent() if err != nil { r.Error(err) return } - info, err := elasticsearch.GetInfo(m.http, m.HostData().SanitizedURI) + info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+statsPath) if err != nil { r.Error(err) return } - eventMapping(r, *info, content) + if m.XPack { + eventMappingXPack(r, m, *info, content) + } else { + eventMapping(r, *info, content) + } + }