diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 0dc12daa61ad..761886d6f8c6 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -41647,6 +41647,154 @@ type: keyword -- +[float] +=== cluster_actions + +Kibana cluster actions metrics. + + + + +*`kibana.cluster_actions.kibana.status`*:: ++ +-- +type: keyword + +-- + + +*`kibana.cluster_actions.overdue.count`*:: ++ +-- +type: long + +-- + + +*`kibana.cluster_actions.overdue.delay.p50`*:: ++ +-- +type: float + +-- + +*`kibana.cluster_actions.overdue.delay.p99`*:: ++ +-- +type: float + +-- + +[float] +=== cluster_rules + +Kibana cluster rule metrics. + + + + +*`kibana.cluster_rules.kibana.status`*:: ++ +-- +type: keyword + +-- + + +*`kibana.cluster_rules.overdue.count`*:: ++ +-- +type: long + +-- + + +*`kibana.cluster_rules.overdue.delay.p50`*:: ++ +-- +type: float + +-- + +*`kibana.cluster_rules.overdue.delay.p99`*:: ++ +-- +type: float + +-- + +[float] +=== node_actions + +Kibana node actions metrics. + + + + +*`kibana.node_actions.kibana.status`*:: ++ +-- +type: keyword + +-- + +*`kibana.node_actions.failures`*:: ++ +-- +type: long + +-- + +*`kibana.node_actions.executions`*:: ++ +-- +type: long + +-- + +*`kibana.node_actions.timeouts`*:: ++ +-- +type: long + +-- + +[float] +=== node_rules + +Kibana node rule metrics. + + + + +*`kibana.node_rules.kibana.status`*:: ++ +-- +type: keyword + +-- + +*`kibana.node_rules.failures`*:: ++ +-- +type: long + +-- + +*`kibana.node_rules.executions`*:: ++ +-- +type: long + +-- + +*`kibana.node_rules.timeouts`*:: ++ +-- +type: long + +-- + [float] === settings diff --git a/metricbeat/docs/modules/kibana.asciidoc b/metricbeat/docs/modules/kibana.asciidoc index 9f609ac91316..a81c1ef6b08d 100644 --- a/metricbeat/docs/modules/kibana.asciidoc +++ b/metricbeat/docs/modules/kibana.asciidoc @@ -61,12 +61,28 @@ It also supports the options described in <>. The following metricsets are available: +* <> + +* <> + +* <> + +* <> + * <> * <> * <> +include::kibana/cluster_actions.asciidoc[] + +include::kibana/cluster_rules.asciidoc[] + +include::kibana/node_actions.asciidoc[] + +include::kibana/node_rules.asciidoc[] + include::kibana/settings.asciidoc[] include::kibana/stats.asciidoc[] diff --git a/metricbeat/docs/modules/kibana/cluster_actions.asciidoc b/metricbeat/docs/modules/kibana/cluster_actions.asciidoc new file mode 100644 index 000000000000..25d6944cc6bd --- /dev/null +++ b/metricbeat/docs/modules/kibana/cluster_actions.asciidoc @@ -0,0 +1,28 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/kibana/cluster_actions/_meta/docs.asciidoc + + +[[metricbeat-metricset-kibana-cluster_actions]] +=== Kibana cluster_actions metricset + +beta[] + +include::../../../module/kibana/cluster_actions/_meta/docs.asciidoc[] + + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kibana/cluster_actions/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules/kibana/cluster_rules.asciidoc b/metricbeat/docs/modules/kibana/cluster_rules.asciidoc new file mode 100644 index 000000000000..e12c4449c002 --- /dev/null +++ b/metricbeat/docs/modules/kibana/cluster_rules.asciidoc @@ -0,0 +1,28 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/kibana/cluster_rules/_meta/docs.asciidoc + + +[[metricbeat-metricset-kibana-cluster_rules]] +=== Kibana cluster_rules metricset + +beta[] + +include::../../../module/kibana/cluster_rules/_meta/docs.asciidoc[] + + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kibana/cluster_rules/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules/kibana/node_actions.asciidoc b/metricbeat/docs/modules/kibana/node_actions.asciidoc new file mode 100644 index 000000000000..e92cf9c07b42 --- /dev/null +++ b/metricbeat/docs/modules/kibana/node_actions.asciidoc @@ -0,0 +1,28 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/kibana/node_actions/_meta/docs.asciidoc + + +[[metricbeat-metricset-kibana-node_actions]] +=== Kibana node_actions metricset + +beta[] + +include::../../../module/kibana/node_actions/_meta/docs.asciidoc[] + + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kibana/node_actions/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules/kibana/node_rules.asciidoc b/metricbeat/docs/modules/kibana/node_rules.asciidoc new file mode 100644 index 000000000000..d116e13a4d04 --- /dev/null +++ b/metricbeat/docs/modules/kibana/node_rules.asciidoc @@ -0,0 +1,28 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/metricbeat/module/kibana/node_rules/_meta/docs.asciidoc + + +[[metricbeat-metricset-kibana-node_rules]] +=== Kibana node_rules metricset + +beta[] + +include::../../../module/kibana/node_rules/_meta/docs.asciidoc[] + + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/kibana/node_rules/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 5e2952bc3ed0..7a088795be23 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -163,7 +163,11 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | -.3+| .3+| |<> +.7+| .7+| |<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] +|<> |<> |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index d57ebf4631d9..6744af0eaa6a 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -88,6 +88,10 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/kafka/consumergroup" _ "github.com/elastic/beats/v7/metricbeat/module/kafka/partition" _ "github.com/elastic/beats/v7/metricbeat/module/kibana" + _ "github.com/elastic/beats/v7/metricbeat/module/kibana/cluster_actions" + _ "github.com/elastic/beats/v7/metricbeat/module/kibana/cluster_rules" + _ "github.com/elastic/beats/v7/metricbeat/module/kibana/node_actions" + _ "github.com/elastic/beats/v7/metricbeat/module/kibana/node_rules" _ "github.com/elastic/beats/v7/metricbeat/module/kibana/settings" _ "github.com/elastic/beats/v7/metricbeat/module/kibana/stats" _ "github.com/elastic/beats/v7/metricbeat/module/kibana/status" diff --git a/metricbeat/module/kibana/cluster_actions/_meta/data.json b/metricbeat/module/kibana/cluster_actions/_meta/data.json new file mode 100644 index 000000000000..6d66640448b0 --- /dev/null +++ b/metricbeat/module/kibana/cluster_actions/_meta/data.json @@ -0,0 +1,25 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"kibana", + "name":"cluster_actions", + "rtt":44269 + }, + "kibana":{ + "cluster_actions":{ + "overdue": { + "count": 3, + "delay": { + "p50": 500, + "p99": 3000 + } + } + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/kibana/cluster_actions/_meta/docs.asciidoc b/metricbeat/module/kibana/cluster_actions/_meta/docs.asciidoc new file mode 100644 index 000000000000..e19a98beb83c --- /dev/null +++ b/metricbeat/module/kibana/cluster_actions/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the actions metricset of the module kibana. diff --git a/metricbeat/module/kibana/cluster_actions/_meta/fields.yml b/metricbeat/module/kibana/cluster_actions/_meta/fields.yml new file mode 100644 index 000000000000..8454312a5687 --- /dev/null +++ b/metricbeat/module/kibana/cluster_actions/_meta/fields.yml @@ -0,0 +1,23 @@ +- name: cluster_actions + type: group + description: > + Kibana cluster actions metrics. + release: beta + fields: + - name: kibana + type: group + fields: + - name: status + type: keyword + - name: overdue + type: group + fields: + - name: count + type: long + - name: delay + type: group + fields: + - name: p50 + type: float + - name: p99 + type: float diff --git a/metricbeat/module/kibana/cluster_actions/cluster_actions.go b/metricbeat/module/kibana/cluster_actions/cluster_actions.go new file mode 100644 index 000000000000..a2f37d4b8de4 --- /dev/null +++ b/metricbeat/module/kibana/cluster_actions/cluster_actions.go @@ -0,0 +1,99 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cluster_actions + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/metricbeat/module/kibana" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet(kibana.ModuleName, "cluster_actions", New, + mb.WithHostParser(hostParser), + ) +} + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + DefaultPath: kibana.ClusterActionsPath, + }.Build() +) + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *kibana.MetricSet + actionsHTTP *helper.HTTP +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := kibana.NewMetricSet(base) + if err != nil { + return nil, err + } + + actionsHTTP, err := helper.NewHTTP(ms.BaseMetricSet) + if err != nil { + return nil, err + } + kibanaVersion, err := kibana.GetVersion(actionsHTTP, kibana.ClusterActionsPath) + if err != nil { + return nil, err + } + + isMetricsAPIAvailable := kibana.IsActionsAPIAvailable(kibanaVersion) + if !isMetricsAPIAvailable { + const errorMsg = "the %v cluster actions is only supported with Kibana >= %v. You are currently running Kibana %v" + return nil, fmt.Errorf(errorMsg, ms.FullyQualifiedName(), kibana.ActionsAPIAvailableVersion, kibanaVersion) + } + + return &MetricSet{ + MetricSet: ms, + actionsHTTP: actionsHTTP, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { + if err = m.fetchMetrics(r); err != nil { + return fmt.Errorf("error trying to get cluster action data from Kibana: %w", err) + } + + return nil +} + +func (m *MetricSet) fetchMetrics(r mb.ReporterV2) error { + var content []byte + var err error + + content, err = m.actionsHTTP.FetchContent() + if err != nil { + return err + } + + return eventMapping(r, content, m.XPackEnabled) +} diff --git a/metricbeat/module/kibana/cluster_actions/data.go b/metricbeat/module/kibana/cluster_actions/data.go new file mode 100644 index 000000000000..182d7325a226 --- /dev/null +++ b/metricbeat/module/kibana/cluster_actions/data.go @@ -0,0 +1,108 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cluster_actions + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/elastic/elastic-agent-libs/mapstr" + + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/kibana" +) + +var ( + actionsSchema = s.Schema{ + "overdue": c.Dict("overdue", s.Schema{ + "count": c.Int("count"), + "delay": c.Dict("delay", s.Schema{ + "p50": c.Float("p50"), + "p99": c.Float("p99"), + }), + }), + } +) + +type response struct { + Actions map[string]interface{} `json:"cluster_actions"` + Kibana map[string]interface{} `json:"kibana"` + ClusterUuid string `json:"cluster_uuid"` +} + +func eventMapping(r mb.ReporterV2, content []byte, isXpack bool) error { + var data response + err := json.Unmarshal(content, &data) + if err != nil { + return fmt.Errorf("failure parsing Kibana Cluster Actions API response: %w", err) + } + + kibanaData, err := kibana.KibanaSchema.Apply(data.Kibana) + if err != nil { + return elastic.MakeErrorForMissingField("kibana", elastic.Kibana) + } + + // Set service ID + serviceId, err := kibanaData.GetValue("uuid") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.uuid", elastic.Kibana) + } + + // Set service version + version, err := kibanaData.GetValue("version") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.version", elastic.Kibana) + } + + // Set service address + serviceAddress, err := kibanaData.GetValue("transport_address") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.transport_address", elastic.Kibana) + } + + actionsFields, err := actionsSchema.Apply(data.Actions) + if err != nil { + return fmt.Errorf("failure to apply cluster actions specific schema: %w", err) + } + + event := mb.Event{ + ModuleFields: mapstr.M{ + "elasticsearch.cluster.id": data.ClusterUuid, + }, + RootFields: mapstr.M{ + "service.id": serviceId, + "service.version": version, + "service.address": serviceAddress, + }, + MetricSetFields: actionsFields, + } + + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Kibana) + event.Index = index + } + + r.Event(event) + + return nil +} diff --git a/metricbeat/module/kibana/cluster_rules/_meta/data.json b/metricbeat/module/kibana/cluster_rules/_meta/data.json new file mode 100644 index 000000000000..8d9a308d4709 --- /dev/null +++ b/metricbeat/module/kibana/cluster_rules/_meta/data.json @@ -0,0 +1,25 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"kibana", + "name":"cluster_rules", + "rtt":44269 + }, + "kibana":{ + "cluster_rules":{ + "overdue": { + "count": 3, + "delay": { + "p50": 500, + "p99": 3000 + } + } + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/kibana/cluster_rules/_meta/docs.asciidoc b/metricbeat/module/kibana/cluster_rules/_meta/docs.asciidoc new file mode 100644 index 000000000000..71d7ab15ab4a --- /dev/null +++ b/metricbeat/module/kibana/cluster_rules/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the rules metricset of the module kibana. diff --git a/metricbeat/module/kibana/cluster_rules/_meta/fields.yml b/metricbeat/module/kibana/cluster_rules/_meta/fields.yml new file mode 100644 index 000000000000..8d96920079fc --- /dev/null +++ b/metricbeat/module/kibana/cluster_rules/_meta/fields.yml @@ -0,0 +1,23 @@ +- name: cluster_rules + type: group + description: > + Kibana cluster rule metrics. + release: beta + fields: + - name: kibana + type: group + fields: + - name: status + type: keyword + - name: overdue + type: group + fields: + - name: count + type: long + - name: delay + type: group + fields: + - name: p50 + type: float + - name: p99 + type: float diff --git a/metricbeat/module/kibana/cluster_rules/cluster_rules.go b/metricbeat/module/kibana/cluster_rules/cluster_rules.go new file mode 100644 index 000000000000..c221ba78ef0e --- /dev/null +++ b/metricbeat/module/kibana/cluster_rules/cluster_rules.go @@ -0,0 +1,100 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cluster_rules + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/metricbeat/module/kibana" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet(kibana.ModuleName, "cluster_rules", New, + mb.WithHostParser(hostParser), + ) +} + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + DefaultPath: kibana.ClusterRulesPath, + }.Build() +) + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *kibana.MetricSet + rulesHTTP *helper.HTTP +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := kibana.NewMetricSet(base) + if err != nil { + return nil, err + } + + rulesHTTP, err := helper.NewHTTP(ms.BaseMetricSet) + if err != nil { + return nil, err + } + + kibanaVersion, err := kibana.GetVersion(rulesHTTP, kibana.ClusterRulesPath) + if err != nil { + return nil, err + } + + isMetricsAPIAvailable := kibana.IsRulesAPIAvailable(kibanaVersion) + if !isMetricsAPIAvailable { + const errorMsg = "the %v cluster rules is only supported with Kibana >= %v. You are currently running Kibana %v" + return nil, fmt.Errorf(errorMsg, ms.FullyQualifiedName(), kibana.RulesAPIAvailableVersion, kibanaVersion) + } + + return &MetricSet{ + MetricSet: ms, + rulesHTTP: rulesHTTP, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { + if err = m.fetchMetrics(r); err != nil { + return fmt.Errorf("error trying to get cluster rule data from Kibana: %w", err) + } + + return nil +} + +func (m *MetricSet) fetchMetrics(r mb.ReporterV2) error { + var content []byte + var err error + + content, err = m.rulesHTTP.FetchContent() + if err != nil { + return err + } + + return eventMapping(r, content, m.XPackEnabled) +} diff --git a/metricbeat/module/kibana/cluster_rules/data.go b/metricbeat/module/kibana/cluster_rules/data.go new file mode 100644 index 000000000000..85b8093d5629 --- /dev/null +++ b/metricbeat/module/kibana/cluster_rules/data.go @@ -0,0 +1,108 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cluster_rules + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/elastic/beats/v7/metricbeat/module/kibana" + "github.com/elastic/elastic-agent-libs/mapstr" + + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +var ( + rulesSchema = s.Schema{ + "overdue": c.Dict("overdue", s.Schema{ + "count": c.Int("count"), + "delay": c.Dict("delay", s.Schema{ + "p50": c.Float("p50"), + "p99": c.Float("p99"), + }), + }), + } +) + +type response struct { + Rules map[string]interface{} `json:"cluster_rules"` + Kibana map[string]interface{} `json:"kibana"` + ClusterUuid string `json:"cluster_uuid"` +} + +func eventMapping(r mb.ReporterV2, content []byte, isXpack bool) error { + var data response + err := json.Unmarshal(content, &data) + if err != nil { + return fmt.Errorf("failure parsing Kibana Cluster Rules API response: %w", err) + } + + kibanaData, err := kibana.KibanaSchema.Apply(data.Kibana) + if err != nil { + return elastic.MakeErrorForMissingField("kibana", elastic.Kibana) + } + + // Set service ID + serviceId, err := kibanaData.GetValue("uuid") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.uuid", elastic.Kibana) + } + + // Set service version + version, err := kibanaData.GetValue("version") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.version", elastic.Kibana) + } + + // Set service address + serviceAddress, err := kibanaData.GetValue("transport_address") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.transport_address", elastic.Kibana) + } + + rulesFields, err := rulesSchema.Apply(data.Rules) + if err != nil { + return fmt.Errorf("failure to apply cluster rules specific schema: %w", err) + } + + event := mb.Event{ + ModuleFields: mapstr.M{ + "elasticsearch.cluster.id": data.ClusterUuid, + }, + RootFields: mapstr.M{ + "service.id": serviceId, + "service.version": version, + "service.address": serviceAddress, + }, + MetricSetFields: rulesFields, + } + + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Kibana) + event.Index = index + } + + r.Event(event) + + return nil +} diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index 2c728c5076d6..1ea7bfe418af 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKibana returns asset data. // This is the base64 encoded zlib format compressed contents of module/kibana. func AssetKibana() string { - return "eJzMWU2P2zYQve+vGPiSS1bAHnLxoWjRFmhRJCjSBj0UhTGWxhYbilQ51O66v74gJdn6oCxZkpv4sFhI4ntvZsgZcvgIn+m0hc9ijwofAKywkraw+cU/2DwAJMSxEbkVWm3hmwcAgPIlZDopJD0AcKqN3cVaHcRxCweU7J4akoRMWzg6YCZrhTryFv7cMMvNW9ik1uabvx4ADoJkwluP/QgKM6oV7diiZf8CwJ5yh2Z0kVdPmgObg63IiC1m+flNPRqlQG48zdGmW9h8ex6x6YGVSiJDnGvFtHOfRhm+TsSuhntD+iBRxkOEbkTBc1jCADW85iijTJtTdDBEO6F2+5OlWURjUDVlbnRMzFGRO7vdd5mQUszibGOF/Fd/UUlLCfOIxb+0kyITdglnGDEKGx1rFRfGkPJLQ1HsltAsk0eQBuw2xCJxY5jszqtdEuox8AEnaI6kxiR6ymbOr+7oLvC7RcDvhoGfliE/BaBrH9Kz85zUOt8lJPG0JB5drNCCMPRPQWw5stqinJe3PEIHoIefCK4m6Kw5VrOEYC5cjQzKd83DHSp8JoNHWk6Hz8crab8oRDKRg8k8i5iiakAb6OaiSRLZipgJTZxGsSzYkokCYj7T6UWbpAdQl/jegCa7+wW2FPWv2lp4twGqBEyhHp3XICNrRMxR4/P2DqP+de1rSuw4d9imgNBKmlBsUcUEnz79/EOQxP1di6SHVZMIldDrXJYPmBHoQ8X2hkEoS0ahDMDWhKlmu5ZVDmvQMmtQce52lJgkhpgXsrpVQuYNn1n9vHIEQfpnMiy0WkgaQjmvE4U5pzrszb3WkrBL36L4IyWbkgGbUk23L4RMQDDgGb18FhbQ3VkuCaZzLKG0aQi2ZpQ6Rjl9UZwLZjdI9SA3X49k+imocVyArzD/tFLzdZFDUF1ze4Gc5tqC27VsjpBruWiShqFs3C13MFTyLr9MHA2WEbWm6Fp2JeJwifp9U/s47X2SfZ93bvqP7mv+ClVhfOKER68xe1asMyEzpokIo7ZdEP7mmgvWqlx92V+4lo1OyknVTYf552T0RLA1+qaM3h/+saxLs1FyifagTbYYYKmQstkwODzkYLji5Cb0UO+rzyK1Ol6F8ofilbAKpmQB1GW/hX2vruG2p9CkuECnKA+7g9TYTRFtlHeroDzNg5ncn4Nrvh4rsUW2J+OKbCwFKQsNCsgwIbDaJ72yakTwQVsCm6KFvdEvTIYhRgVMKoGskFbkkoCF+xcV6YJbiFbXTZjGxx4ZkJncXtaCVjG99UXJpnTy8IYeCyYgtriXglNKmrBReFNedp4mp7wRT/1awtX765s2nhObkaPBPK++Xk958tBrfbg2CLujUDIwv0ecBfCj4wHHA54HhIKymU6xVkmXtOOplLC/8q9lhVE9dfgc8kAMx6uez6ADQRtJna7MoN3C0OBRAwB+d+ylASjdSdVS4lZUNdGdh8PozaT9xdT/5HQ7BbA/TZc8eHvx/wl/j68RaJkA5xiTV9T2/wedUPQ310a9nRCIwfU7Ys4EuZ88tsvoDScPrrxOa3ytZAnwsUrzsS6UvSlptnaJgTb9Wo66lL76XqCsay9kqEFNw9u+7jXFWsrKha56+gYi1+jZz4vfxwqivI+eGazObcEcb3xX3lycTfJ6RgpHo3S0b0fmCHiPryIrsokCBo92N3fzfvMIlYsbb25v3N2/8RPebH21LTInLCy5ai9E5Tq7T3+jWsTRvfocft5E2q0bKf0l3vrRr9A92UD0wxljdvV4X8L11wNMyEOTjkuwQpb+vmRpEA6fSSBwC33zuXeCprrqulgJf0va1zHtAH299sJ4p2CSXghUu2bpve5RmFCL76m0Je6/AAAA//9Bvx66" + return "eJzsWk2P2zYTvu+vGPiSS1Z497CH+PCiRVugRZGgSBv0UBTGWBpbbChS5VCbdX99QX3YskRKsiw3iyA+LBaS5nkezpAz/LqHj3RYw0exRYV3AFZYSWtY/Vw+WN0BJMSxEbkVWq3h/3cAANVLyHRSSLoD4FQbu4m12on9GnYo2T01JAmZ1rB3wEzWCrXnNfyxYpar17BKrc1Xf94B7ATJhNcl9j0ozKhRtGGLlssXAPaQOzSji7x+0jZsG1uREVvM8uObxhqlQG49zdGma1h9c7RY9cAqJZEhzrVi2rhPowyfJ2LX5mVD+iBRxiFCZ1HwHBY/QAOvOcoo0+YQ7QzRRqjN9mBpFtEYVEOZGx0Tc1Tkrt3uu0xIKWZxnmP5/Nd8UUtLCfOIxT+0kSIT9hpOP2Lkb3SsVVwYQ6ocGopiN4RmNXkEKdBuQywSZ8NkN6Xaa0I9Bh5wguZIakyih2xm/+pad4EfrwJ+DAM/XIf84IFufEhPznNS63yTkMTDNfHoYvkGhKG/C2LLkdUW5by8VSJ0AHr4ieC6g87qYw2LD+bE1cqgfNM83KHCJzK4p+vp8Gk/kPaLQiQTOZjMk4gpqg3OgS4umiSRrYiZ0MRpFMuCLZnII+YjHT5pk/QTXmWywUCma4twP8/MovnVM4waEWpEyMgaEXPU+vY4y9iSxdbzbiuDDhrWGIJqw/UK9bC72qb6iUxS0GJSYl0oG1QitdoHTbuZaFzMkKA2cv74P+/7BnsnNfpEtxDevLkAodsfTSE9NW9+b3R4X7vi1644AaGxUjqhBfOig/tyk+IOhSwMdY0DHedYvp4pLroOnmDnSrIu7ASrs2AulVTKUH6ZGeWlx7HZklggiuU0D1AlYAp176QMhnM/NZidyeCw3ztCa2lCsUUVE3z48NP3XhL3dymSHlZDIlRCz3NZ3mFGoHc12ysGoSwZhdID2xCmmruZfHarHFawZdag4lwbu8EkMcT+Hjid1c3qybziI2vZrxyBl/6JDAutriT1oRzHicKcU+335lZrSdilP6P4PSWbkgGbUkO3LYRMQDDgEb165hfgy0Fzg+kcSyht6oNtGKWOUU4fFMfJQTdIjZHrr3sy/RTU2t6EF5h/Xk4xKfh87T1HyFAumqQhlI27y3MILdFPv0zsDVYRtaY37R6IOJyiftvUPk57m2Tf552b/qPbNn+BqjDecfzWS/SeBeuMrxnTRPhRz13g/2bIBUtVrr7sz1zLRjvlpOqm/fxzMnoi2Bp9UUbvm7+v6tJslFyi3WmTXQ1wrZDqcCRofs3OReisrs/i3VppQ5Wb+AthFUzJFVCn+Rb2vbqE2x58neIEnaLcbcY3fR4XQXmYBzP5PBGGfD1WYotsS8YV2VgKUhZaFJBhQmB1mfSqqhHBO20JbIoWtkZ/YjIMMSpgUglkhbQilwQs3L+oSBd8hmh1c2jU+rhEBmQmN5e1oFVMr8uiZFM6lPCG7gsmILa4lYJTStqwkX9SXp2UTU55I576pYJr5tcXTTwnHp6OBvM4+npn4JNNh84Nz0HYLYWSQP8ecRbAD44HHE+1rQtCQXX4T7FWSZe046mUsD/yh7LCqJ4mfA45EMPxqldm0EDQRlKnKzNo1xAyHm0AwG+OvWoASrdStZS4EVV3dOdhP3o7aX829T863U4BbA/TJQdvW/x3wt/icwRaJsA5xlQqOvf/O51Q9Bc3jXo9IRDB8TvSnAlyP5TYLqO3nBwceZ2j/KWSJcD7Os2Xx0EXJc2zWaLnWsFSjjqVvuYeQ1XXPpGhFjWFp33daxVLKasGuurpC0SudcdgXvze1xDV/bmZwercbpjjjW+rmxbHJpV6RgpHq3Sc3+aYI+AtPousyCYKCC7tLt7N+7VEqF3cenP5xt3tN378k60Xu0XmhPkl19sLUTXObrO/UQ/i6Fb7HGW/ibQbN1KWl46Wj36NXpIFou/PGLOrx9sKrj8eYNL9gwnLJVggS39XsbQIw2sS8Nyau3jdO0FTU3VdrER5q6uvY9oCerj2wvhOwSS94Kl27dI77FGYUItvqfRM3L8BAAD//0ftJ4w=" } diff --git a/metricbeat/module/kibana/kibana.go b/metricbeat/module/kibana/kibana.go index 989289e1b58a..781a0fccfda9 100644 --- a/metricbeat/module/kibana/kibana.go +++ b/metricbeat/module/kibana/kibana.go @@ -26,6 +26,9 @@ import ( "github.com/elastic/beats/v7/metricbeat/helper/elastic" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/elastic-agent-libs/logp" + + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" "github.com/elastic/elastic-agent-libs/version" ) @@ -34,9 +37,13 @@ const ( ModuleName = "kibana" // API Paths - StatusPath = "api/status" - StatsPath = "api/stats" - SettingsPath = "api/settings" + StatusPath = "api/status" + StatsPath = "api/stats" + ClusterRulesPath = "api/monitoring_collection/cluster_rules" + NodeRulesPath = "api/monitoring_collection/node_rules" + ClusterActionsPath = "api/monitoring_collection/cluster_actions" + NodeActionsPath = "api/monitoring_collection/node_actions" + SettingsPath = "api/settings" ) var ( @@ -45,12 +52,32 @@ var ( v6_7_2 = version.MustNew("6.7.2") v7_0_0 = version.MustNew("7.0.0") v7_0_1 = version.MustNew("7.0.1") + v8_2_0 = version.MustNew("8.2.0") // StatsAPIAvailableVersion is the version of Kibana since when the stats API is available StatsAPIAvailableVersion = v6_4_0 // SettingsAPIAvailableVersion is the version of Kibana since when the settings API is available SettingsAPIAvailableVersion = v6_5_0 + + // Version of Kibana since when the rules and task manager APIs are available + RulesAPIAvailableVersion = v8_2_0 + ActionsAPIAvailableVersion = v8_2_0 +) + +var ( + KibanaSchema = s.Schema{ + "uuid": c.Str("uuid"), + "name": c.Str("name"), + "index": c.Str("index"), + "host": s.Object{ + "name": c.Str("host"), + }, + "transport_address": c.Str("transport_address"), + "version": c.Str("version"), + "snapshot": c.Bool("snapshot"), + "status": c.Str("status"), + } ) func init() { @@ -62,7 +89,10 @@ func init() { // NewModule creates a new module. func NewModule(base mb.BaseModule) (mb.Module, error) { - return elastic.NewModule(&base, []string{"stats"}, logp.NewLogger(ModuleName)) + xpackEnabledMetricSets := []string{ + "stats", "cluster_rules", "node_rules", "cluster_actions", "node_actions", + } + return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName)) } // GetVersion returns the version of the Kibana instance @@ -96,6 +126,16 @@ func IsSettingsAPIAvailable(currentKibanaVersion *version.V) bool { return elastic.IsFeatureAvailable(currentKibanaVersion, SettingsAPIAvailableVersion) } +// IsRulesAPIAvailable returns whether the rules API is available in the given version of Kibana +func IsRulesAPIAvailable(currentKibanaVersion *version.V) bool { + return elastic.IsFeatureAvailable(currentKibanaVersion, RulesAPIAvailableVersion) +} + +// IsActionsAPIAvailable returns whether the actions API is available in the given version of Kibana +func IsActionsAPIAvailable(currentKibanaVersion *version.V) bool { + return elastic.IsFeatureAvailable(currentKibanaVersion, ActionsAPIAvailableVersion) +} + // IsUsageExcludable returns whether the stats API supports the exclude_usage parameter in the // given version of Kibana func IsUsageExcludable(currentKibanaVersion *version.V) bool { diff --git a/metricbeat/module/kibana/mtest/testing.go b/metricbeat/module/kibana/mtest/testing.go index 7a96f4c7a3f6..b8eb756442ad 100644 --- a/metricbeat/module/kibana/mtest/testing.go +++ b/metricbeat/module/kibana/mtest/testing.go @@ -18,15 +18,12 @@ package mtest // GetConfig returns config for kibana module -func GetConfig(metricset string, host string, xpackEnabled bool) map[string]interface{} { +func GetConfig(metricset string, host string) map[string]interface{} { config := map[string]interface{}{ "module": "kibana", "metricsets": []string{metricset}, "hosts": []string{host}, } - if xpackEnabled { - config["xpack.enabled"] = true - } return config } diff --git a/metricbeat/module/kibana/node_actions/_meta/data.json b/metricbeat/module/kibana/node_actions/_meta/data.json new file mode 100644 index 000000000000..3a49b2ea5914 --- /dev/null +++ b/metricbeat/module/kibana/node_actions/_meta/data.json @@ -0,0 +1,21 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"kibana", + "name":"node_actions", + "rtt":44269 + }, + "kibana":{ + "node_actions":{ + "executions": 10, + "failures": 0, + "timeouts": 0 + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/kibana/node_actions/_meta/docs.asciidoc b/metricbeat/module/kibana/node_actions/_meta/docs.asciidoc new file mode 100644 index 000000000000..e19a98beb83c --- /dev/null +++ b/metricbeat/module/kibana/node_actions/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the actions metricset of the module kibana. diff --git a/metricbeat/module/kibana/node_actions/_meta/fields.yml b/metricbeat/module/kibana/node_actions/_meta/fields.yml new file mode 100644 index 000000000000..155101b24890 --- /dev/null +++ b/metricbeat/module/kibana/node_actions/_meta/fields.yml @@ -0,0 +1,17 @@ +- name: node_actions + type: group + description: > + Kibana node actions metrics. + release: beta + fields: + - name: kibana + type: group + fields: + - name: status + type: keyword + - name: failures + type: long + - name: executions + type: long + - name: timeouts + type: long diff --git a/metricbeat/module/kibana/node_actions/data.go b/metricbeat/module/kibana/node_actions/data.go new file mode 100644 index 000000000000..ead19bb20528 --- /dev/null +++ b/metricbeat/module/kibana/node_actions/data.go @@ -0,0 +1,104 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package node_actions + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/elastic/beats/v7/metricbeat/module/kibana" + "github.com/elastic/elastic-agent-libs/mapstr" + + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +var ( + actionsSchema = s.Schema{ + "failures": c.Int("failures"), + "executions": c.Int("executions"), + "timeouts": c.Int("timeouts"), + } +) + +type response struct { + Actions map[string]interface{} `json:"node_actions"` + Kibana map[string]interface{} `json:"kibana"` + ClusterUuid string `json:"cluster_uuid"` +} + +func eventMapping(r mb.ReporterV2, content []byte, isXpack bool) error { + var data response + err := json.Unmarshal(content, &data) + if err != nil { + return fmt.Errorf("failure parsing Kibana Node Actions API response: %w", err) + } + + kibanaData, err := kibana.KibanaSchema.Apply(data.Kibana) + if err != nil { + return elastic.MakeErrorForMissingField("kibana", elastic.Kibana) + } + + // Set service ID + serviceId, err := kibanaData.GetValue("uuid") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.uuid", elastic.Kibana) + } + + // Set service version + version, err := kibanaData.GetValue("version") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.version", elastic.Kibana) + } + + // Set service address + serviceAddress, err := kibanaData.GetValue("transport_address") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.transport_address", elastic.Kibana) + } + + actionsFields, err := actionsSchema.Apply(data.Actions) + if err != nil { + return fmt.Errorf("failure to apply node actions specific schema: %w", err) + } + + event := mb.Event{ + ModuleFields: mapstr.M{ + "elasticsearch.cluster.id": data.ClusterUuid, + }, + RootFields: mapstr.M{ + "service.id": serviceId, + "service.version": version, + "service.address": serviceAddress, + }, + MetricSetFields: actionsFields, + } + + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Kibana) + event.Index = index + } + + r.Event(event) + + return nil +} diff --git a/metricbeat/module/kibana/node_actions/node_actions.go b/metricbeat/module/kibana/node_actions/node_actions.go new file mode 100644 index 000000000000..48324df632f8 --- /dev/null +++ b/metricbeat/module/kibana/node_actions/node_actions.go @@ -0,0 +1,100 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package node_actions + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/metricbeat/module/kibana" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet(kibana.ModuleName, "node_actions", New, + mb.WithHostParser(hostParser), + ) +} + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + DefaultPath: kibana.NodeActionsPath, + }.Build() +) + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *kibana.MetricSet + actionsHTTP *helper.HTTP +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := kibana.NewMetricSet(base) + if err != nil { + return nil, err + } + + actionsHTTP, err := helper.NewHTTP(ms.BaseMetricSet) + if err != nil { + return nil, err + } + + kibanaVersion, err := kibana.GetVersion(actionsHTTP, kibana.NodeActionsPath) + if err != nil { + return nil, err + } + + isMetricsAPIAvailable := kibana.IsActionsAPIAvailable(kibanaVersion) + if !isMetricsAPIAvailable { + const errorMsg = "the %v node actions is only supported with Kibana >= %v. You are currently running Kibana %v" + return nil, fmt.Errorf(errorMsg, ms.FullyQualifiedName(), kibana.ActionsAPIAvailableVersion, kibanaVersion) + } + + return &MetricSet{ + MetricSet: ms, + actionsHTTP: actionsHTTP, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { + if err = m.fetchMetrics(r); err != nil { + return fmt.Errorf("error trying to get node action data from Kibana: %w", err) + } + + return nil +} + +func (m *MetricSet) fetchMetrics(r mb.ReporterV2) error { + var content []byte + var err error + + content, err = m.actionsHTTP.FetchContent() + if err != nil { + return err + } + + return eventMapping(r, content, m.XPackEnabled) +} diff --git a/metricbeat/module/kibana/node_rules/_meta/data.json b/metricbeat/module/kibana/node_rules/_meta/data.json new file mode 100644 index 000000000000..398cd6c79127 --- /dev/null +++ b/metricbeat/module/kibana/node_rules/_meta/data.json @@ -0,0 +1,21 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"kibana", + "name":"node_rules", + "rtt":44269 + }, + "kibana":{ + "node_rules":{ + "executions": 10, + "failures": 0, + "timeouts": 0 + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/kibana/node_rules/_meta/docs.asciidoc b/metricbeat/module/kibana/node_rules/_meta/docs.asciidoc new file mode 100644 index 000000000000..1a84c7075c1b --- /dev/null +++ b/metricbeat/module/kibana/node_rules/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the node_rules metricset of the module kibana. diff --git a/metricbeat/module/kibana/node_rules/_meta/fields.yml b/metricbeat/module/kibana/node_rules/_meta/fields.yml new file mode 100644 index 000000000000..1eaa96f5ffd4 --- /dev/null +++ b/metricbeat/module/kibana/node_rules/_meta/fields.yml @@ -0,0 +1,17 @@ +- name: node_rules + type: group + description: > + Kibana node rule metrics. + release: beta + fields: + - name: kibana + type: group + fields: + - name: status + type: keyword + - name: failures + type: long + - name: executions + type: long + - name: timeouts + type: long diff --git a/metricbeat/module/kibana/node_rules/data.go b/metricbeat/module/kibana/node_rules/data.go new file mode 100644 index 000000000000..6ded4d3bf392 --- /dev/null +++ b/metricbeat/module/kibana/node_rules/data.go @@ -0,0 +1,104 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package node_rules + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/elastic/beats/v7/metricbeat/module/kibana" + "github.com/elastic/elastic-agent-libs/mapstr" + + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +var ( + rulesSchema = s.Schema{ + "failures": c.Int("failures"), + "executions": c.Int("executions"), + "timeouts": c.Int("timeouts"), + } +) + +type response struct { + Rules map[string]interface{} `json:"node_rules"` + Kibana map[string]interface{} `json:"kibana"` + ClusterUuid string `json:"cluster_uuid"` +} + +func eventMapping(r mb.ReporterV2, content []byte, isXpack bool) error { + var data response + err := json.Unmarshal(content, &data) + if err != nil { + return fmt.Errorf("failure parsing Kibana Node Rules API response: %w", err) + } + + kibana, err := kibana.KibanaSchema.Apply(data.Kibana) + if err != nil { + return elastic.MakeErrorForMissingField("kibana", elastic.Kibana) + } + + // Set service ID + serviceId, err := kibana.GetValue("uuid") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.uuid", elastic.Kibana) + } + + // Set service version + version, err := kibana.GetValue("version") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.version", elastic.Kibana) + } + + // Set service address + serviceAddress, err := kibana.GetValue("transport_address") + if err != nil { + return elastic.MakeErrorForMissingField("kibana.transport_address", elastic.Kibana) + } + + rulesFields, err := rulesSchema.Apply(data.Rules) + if err != nil { + return fmt.Errorf("failure to apply node_rules specific schema: %w", err) + } + + event := mb.Event{ + ModuleFields: mapstr.M{ + "elasticsearch.cluster.id": data.ClusterUuid, + }, + RootFields: mapstr.M{ + "service.id": serviceId, + "service.version": version, + "service.address": serviceAddress, + }, + MetricSetFields: rulesFields, + } + + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Kibana) + event.Index = index + } + + r.Event(event) + + return nil +} diff --git a/metricbeat/module/kibana/node_rules/node_rules.go b/metricbeat/module/kibana/node_rules/node_rules.go new file mode 100644 index 000000000000..929c027ac6e7 --- /dev/null +++ b/metricbeat/module/kibana/node_rules/node_rules.go @@ -0,0 +1,100 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package node_rules + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/metricbeat/module/kibana" +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet(kibana.ModuleName, "node_rules", New, + mb.WithHostParser(hostParser), + ) +} + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + DefaultPath: kibana.NodeRulesPath, + }.Build() +) + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + *kibana.MetricSet + rulesHTTP *helper.HTTP +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := kibana.NewMetricSet(base) + if err != nil { + return nil, err + } + + rulesHTTP, err := helper.NewHTTP(ms.BaseMetricSet) + if err != nil { + return nil, err + } + + kibanaVersion, err := kibana.GetVersion(rulesHTTP, kibana.NodeRulesPath) + if err != nil { + return nil, err + } + + isMetricsAPIAvailable := kibana.IsRulesAPIAvailable(kibanaVersion) + if !isMetricsAPIAvailable { + const errorMsg = "the %v node_rules is only supported with Kibana >= %v. You are currently running Kibana %v" + return nil, fmt.Errorf(errorMsg, ms.FullyQualifiedName(), kibana.RulesAPIAvailableVersion, kibanaVersion) + } + + return &MetricSet{ + MetricSet: ms, + rulesHTTP: rulesHTTP, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +// It returns the event which is then forward to the output. In case of an error, a +// descriptive error must be returned. +func (m *MetricSet) Fetch(r mb.ReporterV2) (err error) { + if err = m.fetchMetrics(r); err != nil { + return fmt.Errorf("error trying to get node rule data from Kibana: %w", err) + } + + return nil +} + +func (m *MetricSet) fetchMetrics(r mb.ReporterV2) error { + var content []byte + var err error + + content, err = m.rulesHTTP.FetchContent() + if err != nil { + return err + } + + return eventMapping(r, content, m.XPackEnabled) +} diff --git a/metricbeat/module/kibana/settings/settings_test.go b/metricbeat/module/kibana/settings/settings_test.go index 63280952312b..23bafe356990 100644 --- a/metricbeat/module/kibana/settings/settings_test.go +++ b/metricbeat/module/kibana/settings/settings_test.go @@ -37,7 +37,10 @@ func TestFetchExcludeUsage(t *testing.T) { kib := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/api/status": - w.Write([]byte("{ \"version\": { \"number\": \"7.5.0\" }}")) + _, err := w.Write([]byte("{ \"version\": { \"number\": \"7.5.0\" }}")) + if err != nil { + t.Fatal("write", err) + } case "/api/stats": excludeUsage := r.FormValue("exclude_usage") @@ -62,7 +65,7 @@ func TestFetchExcludeUsage(t *testing.T) { })) defer kib.Close() - config := mtest.GetConfig("settings", kib.URL, false) + config := mtest.GetConfig("settings", kib.URL) f := mbtest.NewReportingMetricSetV2Error(t, config) diff --git a/metricbeat/module/kibana/stats/stats_integration_test.go b/metricbeat/module/kibana/stats/stats_integration_test.go index 33858c6d4b9a..21652a848008 100644 --- a/metricbeat/module/kibana/stats/stats_integration_test.go +++ b/metricbeat/module/kibana/stats/stats_integration_test.go @@ -40,7 +40,7 @@ import ( func TestFetch(t *testing.T) { service := compose.EnsureUpWithTimeout(t, 570, "kibana") - config := mtest.GetConfig("stats", service.Host(), false) + config := mtest.GetConfig("stats", service.Host()) host := config["hosts"].([]string)[0] version, err := getKibanaVersion(t, host) require.NoError(t, err) @@ -65,7 +65,7 @@ func TestFetch(t *testing.T) { func TestData(t *testing.T) { service := compose.EnsureUp(t, "kibana") - config := mtest.GetConfig("stats", service.Host(), false) + config := mtest.GetConfig("stats", service.Host()) host := config["hosts"].([]string)[0] version, err := getKibanaVersion(t, host) require.NoError(t, err) diff --git a/metricbeat/module/kibana/stats/stats_test.go b/metricbeat/module/kibana/stats/stats_test.go index 9de3a7c8c52c..e41addbea6c0 100644 --- a/metricbeat/module/kibana/stats/stats_test.go +++ b/metricbeat/module/kibana/stats/stats_test.go @@ -37,7 +37,10 @@ func TestFetchExcludeUsage(t *testing.T) { kib := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/api/status": - w.Write([]byte("{ \"version\": { \"number\": \"7.5.0\" }}")) + _, err := w.Write([]byte("{ \"version\": { \"number\": \"8.2.0\" }}")) + if err != nil { + t.Fatal("write", err) + } case "/api/stats": excludeUsage := r.FormValue("exclude_usage") @@ -62,7 +65,7 @@ func TestFetchExcludeUsage(t *testing.T) { })) defer kib.Close() - config := mtest.GetConfig("stats", kib.URL, true) + config := mtest.GetConfig("stats", kib.URL) f := mbtest.NewReportingMetricSetV2Error(t, config) @@ -81,7 +84,10 @@ func TestFetchNoExcludeUsage(t *testing.T) { kib := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/api/status": - w.Write([]byte("{ \"version\": { \"number\": \"7.0.0\" }}")) // v7.0.0 does not support exclude_usage and should not be sent + _, err := w.Write([]byte("{ \"version\": { \"number\": \"7.0.0\" }}")) // v7.0.0 does not support exclude_usage and should not be sent + if err != nil { + t.Fatal("write", err) + } case "/api/stats": excludeUsage := r.FormValue("exclude_usage") @@ -91,7 +97,7 @@ func TestFetchNoExcludeUsage(t *testing.T) { })) defer kib.Close() - config := mtest.GetConfig("stats", kib.URL, true) + config := mtest.GetConfig("stats", kib.URL) f := mbtest.NewReportingMetricSetV2Error(t, config) diff --git a/metricbeat/module/kibana/status/status_integration_test.go b/metricbeat/module/kibana/status/status_integration_test.go index 4ae08681bd67..192949ad3053 100644 --- a/metricbeat/module/kibana/status/status_integration_test.go +++ b/metricbeat/module/kibana/status/status_integration_test.go @@ -33,7 +33,7 @@ import ( func TestFetch(t *testing.T) { service := compose.EnsureUpWithTimeout(t, 570, "kibana") - f := mbtest.NewReportingMetricSetV2Error(t, mtest.GetConfig("status", service.Host(), false)) + f := mbtest.NewReportingMetricSetV2Error(t, mtest.GetConfig("status", service.Host())) events, errs := mbtest.ReportingFetchV2Error(f) require.Empty(t, errs)