From 1c6ac3cbae6062f49a46008cfc2bdbd8eab53456 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 27 Aug 2018 17:59:04 -0700 Subject: [PATCH 1/4] Adding index_recovery xpack monitoring documents --- .../index_recovery/data_xpack.go | 87 +++++++++++++++++++ .../index_recovery/index_recovery.go | 12 ++- 2 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 metricbeat/module/elasticsearch/index_recovery/data_xpack.go diff --git a/metricbeat/module/elasticsearch/index_recovery/data_xpack.go b/metricbeat/module/elasticsearch/index_recovery/data_xpack.go new file mode 100644 index 000000000000..9794a03bc4ce --- /dev/null +++ b/metricbeat/module/elasticsearch/index_recovery/data_xpack.go @@ -0,0 +1,87 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package index_recovery + +import ( + "encoding/json" + "time" + + "github.com/elastic/beats/metricbeat/helper/elastic" + "github.com/elastic/beats/metricbeat/module/elasticsearch" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" +) + +func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { + var data map[string]interface{} + err := json.Unmarshal(content, &data) + if err != nil { + return err + } + + var results []map[string]interface{} + for indexName, value := range data { + indexData, ok := value.(map[string]interface{}) + if !ok { + continue + } + + value, ok = indexData["shards"] + if !ok { + continue + } + + shards, ok := value.([]interface{}) + if !ok { + continue + } + + for _, value = range shards { + shard, ok := value.(map[string]interface{}) + if !ok { + continue + } + + shard["index_name"] = indexName + results = append(results, shard) + } + } + + indexRecovery := common.MapStr{} + indexRecovery["shards"] = results + + info, err := elasticsearch.GetInfo(m.HTTP, m.HTTP.GetURI()) + if err != nil { + return err + } + + event := mb.Event{} + event.RootFields = common.MapStr{ + "cluster_uuid": info.ClusterID, + "timestamp": common.Time(time.Now()), + "interval_ms": m.Module().Config().Period / time.Millisecond, + "type": "index_recovery", + "index_recovery": indexRecovery, + } + + event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + r.Event(event) + + return nil +} diff --git a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go index 769839a641ff..0eacb96d6b40 100644 --- a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go +++ b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go @@ -50,7 +50,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { config := struct { ActiveOnly bool `config:"index_recovery.active_only"` }{ - ActiveOnly: true, + ActiveOnly: false, } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err @@ -89,8 +89,12 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } - err = eventsMapping(r, content) - if err != nil { - r.Error(err) + if m.MetricSet.XPack { + eventsMappingXPack(r, m, content) + } else { + err = eventsMapping(r, content) + if err != nil { + r.Error(err) + } } } From 4817f4887741e303faed5a598ffb1d86a24eb0f9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 27 Aug 2018 18:10:23 -0700 Subject: [PATCH 2/4] Get all recoveries if xpack.enabled is true --- .../module/elasticsearch/index_recovery/index_recovery.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go index 0eacb96d6b40..890682b56646 100644 --- a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go +++ b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go @@ -49,15 +49,17 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { config := struct { ActiveOnly bool `config:"index_recovery.active_only"` + XPack bool `config:"xpack.enabled"` }{ - ActiveOnly: false, + ActiveOnly: true, + XPack: false, } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } localRecoveryPath := recoveryPath - if config.ActiveOnly { + if !config.XPack && config.ActiveOnly { localRecoveryPath = localRecoveryPath + "?active_only=true" } From cf7d84b0764e20356d7bc2c8cd75d152b291c99b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 28 Aug 2018 06:03:23 -0700 Subject: [PATCH 3/4] Removing newline (which also reordered imports) --- metricbeat/module/elasticsearch/index_recovery/data_xpack.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/elasticsearch/index_recovery/data_xpack.go b/metricbeat/module/elasticsearch/index_recovery/data_xpack.go index 9794a03bc4ce..87c42252da87 100644 --- a/metricbeat/module/elasticsearch/index_recovery/data_xpack.go +++ b/metricbeat/module/elasticsearch/index_recovery/data_xpack.go @@ -21,11 +21,10 @@ import ( "encoding/json" "time" - "github.com/elastic/beats/metricbeat/helper/elastic" - "github.com/elastic/beats/metricbeat/module/elasticsearch" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/helper/elastic" "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/elasticsearch" ) func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { From 82b9c7fd5fbd4eae123135ea089b457a035699b9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 28 Aug 2018 06:05:51 -0700 Subject: [PATCH 4/4] Don't reuse value so much --- .../elasticsearch/index_recovery/data_xpack.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/elasticsearch/index_recovery/data_xpack.go b/metricbeat/module/elasticsearch/index_recovery/data_xpack.go index 87c42252da87..c0bfae883ee1 100644 --- a/metricbeat/module/elasticsearch/index_recovery/data_xpack.go +++ b/metricbeat/module/elasticsearch/index_recovery/data_xpack.go @@ -35,24 +35,24 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { } var results []map[string]interface{} - for indexName, value := range data { - indexData, ok := value.(map[string]interface{}) + for indexName, indexData := range data { + indexData, ok := indexData.(map[string]interface{}) if !ok { continue } - value, ok = indexData["shards"] + shards, ok := indexData["shards"] if !ok { continue } - shards, ok := value.([]interface{}) + shardsArr, ok := shards.([]interface{}) if !ok { continue } - for _, value = range shards { - shard, ok := value.(map[string]interface{}) + for _, shard := range shardsArr { + shard, ok := shard.(map[string]interface{}) if !ok { continue }