Task/use etcd metrics endpoint

metricbeat/docs/fields.asciidoc

@@ -9582,6 +9582,191 @@ type: keyword
 
 ID of actual leader
 
+--
+
+[float]
+== metrics fields
+
+Etcd metrics from the /metrics endpoint
+
+
+
+*`etcd.metrics.server.has_leader`*::
++
+--
+type: byte
+
+Whether a leader exists in the cluster
+
+
+--
+
+*`etcd.metrics.server.leader_changes.count`*::
++
+--
+type: long
+
+Number of leader changes seen at the cluster
+
+
+--
+
+*`etcd.metrics.server.proposals_committed.count`*::
++
+--
+type: long
+
+Number of consensus proposals commited
+
+
+--
+
+*`etcd.metrics.server.proposals_pending.count`*::
++
+--
+type: long
+
+Number of consensus proposals pending
+
+
+--
+
+*`etcd.metrics.server.proposals_failed.count`*::
++
+--
+type: long
+
+Number of consensus proposals failed
+
+
+--
+
+*`etcd.metrics.server.grpc_started.count`*::
++
+--
+type: long
+
+Number of sent gRPC requests
+
+
+--
+
+*`etcd.metrics.server.grpc_handled.count`*::
++
+--
+type: long
+
+Number of received gRPC requests
+
+
+--
+
+*`etcd.metrics.disk.mvcc_db_total_size.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Size of stored data at MVCC
+
+
+--
+
+*`etcd.metrics.disk.wal_fsync_duration.ns.bucket.*`*::
++
+--
+type: object
+
+Latency for writing ahead logs to disk
+
+
+--
+
+*`etcd.metrics.disk.wal_fsync_duration.ns.count`*::
++
+--
+type: long
+
+Write ahead logs count
+
+
+--
+
+*`etcd.metrics.disk.wal_fsync_duration.ns.sum`*::
++
+--
+type: long
+
+Write ahead logs latency sum
+
+
+--
+
+*`etcd.metrics.disk.backend_commit_duration.ns.bucket.*`*::
++
+--
+type: object
+
+Latency for writing backend changes to disk
+
+
+--
+
+*`etcd.metrics.disk.backend_commit_duration.ns.count`*::
++
+--
+type: long
+
+Backend commits count
+
+
+--
+
+*`etcd.metrics.disk.backend_commit_duration.ns.sum`*::
++
+--
+type: long
+
+Backend commits latency sum
+
+
+--
+
+*`etcd.metrics.memory.go_memstats_alloc.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Memory allocated bytes as of MemStats Go
+
+
+--
+
+*`etcd.metrics.network.client_grpc_sent.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+gRPC sent bytes total
+
+
+--
+
+*`etcd.metrics.network.client_grpc_received.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+gRPC received bytes total
+
+
 --
 
 [float]

metricbeat/docs/modules/etcd.asciidoc

@@ -5,9 +5,13 @@ This file is generated! See scripts/docs_collector.py
 [[metricbeat-module-etcd]]
 == Etcd module
 
-This is the Etcd Module. The Etcd module uses https://coreos.com/etcd/docs/latest/v2/api.html [Etcd v2 API] to collect metrics.
+This module targets Etcd V2 and V3.
 
-The default metricsets are `leader`, `self` and `store`.
+When using V2, metrics are collected using https://coreos.com/etcd/docs/latest/v2/api.html[Etcd v2 API].
+When using V3, metrics are retrieved from the `/metrics` endpoint as intended for https://coreos.com/etcd/docs/latest/metrics.html[Etcd v3]
+
+When using V3, metricsest are bundled into `metrics`
+When using V2, metricsets available are `leader`, `self` and `store`.
 
 [float]
 === Compatibility
@@ -40,12 +44,16 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-etcd-leader,leader>>
 
+* <<metricbeat-metricset-etcd-metrics,metrics>>
+
 * <<metricbeat-metricset-etcd-self,self>>
 
 * <<metricbeat-metricset-etcd-store,store>>
 
 include::etcd/leader.asciidoc[]
 
+include::etcd/metrics.asciidoc[]
+
 include::etcd/self.asciidoc[]
 
 include::etcd/store.asciidoc[]

metricbeat/docs/modules/etcd/metrics.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-etcd-metrics]]
+=== Etcd metrics metricset
+
+beta[]
+
+include::../../../module/etcd/metrics/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-etcd,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/etcd/metrics/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -58,7 +58,8 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-module-envoyproxy,envoyproxy>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-envoyproxy-server,server>>   
 |<<metricbeat-module-etcd,Etcd>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
-.3+| .3+|  |<<metricbeat-metricset-etcd-leader,leader>>   
+.4+| .4+|  |<<metricbeat-metricset-etcd-leader,leader>>   
+|<<metricbeat-metricset-etcd-metrics,metrics>> beta[]  
 |<<metricbeat-metricset-etcd-self,self>>   
 |<<metricbeat-metricset-etcd-store,store>>   
 |<<metricbeat-module-golang,Golang>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  

metricbeat/helper/prometheus/prometheus.go

@@ -93,9 +93,12 @@ func (p *prometheus) GetFamilies() ([]*dto.MetricFamily, error) {
 
 // MetricsMapping defines mapping settings for Prometheus metrics, to be used with `GetProcessedMetrics`
 type MetricsMapping struct {
-	// Metrics translates from from prometheus metric name to Metricbeat fields
+	// Metrics translates from prometheus metric name to Metricbeat fields
 	Metrics map[string]MetricMap
 
+	// Namespace for metrics managed by this mapping
+	Namespace string
+
 	// Labels translate from prometheus label names to Metricbeat fields
 	Labels map[string]LabelMap
 
@@ -213,7 +216,10 @@ func (p *prometheus) ReportProcessedMetrics(mapping *MetricsMapping, r mb.Report
 		return
 	}
 	for _, event := range events {
-		r.Event(mb.Event{MetricSetFields: event})
+		r.Event(mb.Event{
+			MetricSetFields: event,
+			Namespace:       mapping.Namespace,
+		})
 	}
 }
 

metricbeat/module/etcd/_meta/config.yml

@@ -3,6 +3,7 @@
   #  - leader
   #  - self
   #  - store
+  #  - metrics
   period: 10s
   hosts: ["localhost:2379"]
   #username: "user"

metricbeat/module/etcd/_meta/docs.asciidoc

@@ -1,6 +1,10 @@
-This is the Etcd Module. The Etcd module uses https://coreos.com/etcd/docs/latest/v2/api.html [Etcd v2 API] to collect metrics.
+This module targets Etcd V2 and V3.
 
-The default metricsets are `leader`, `self` and `store`.
+When using V2, metrics are collected using https://coreos.com/etcd/docs/latest/v2/api.html[Etcd v2 API].
+When using V3, metrics are retrieved from the `/metrics` endpoint as intended for https://coreos.com/etcd/docs/latest/metrics.html[Etcd v3]
+
+When using V3, metricsest are bundled into `metrics`
+When using V2, metricsets available are `leader`, `self` and `store`.
 
 [float]
 === Compatibility

metricbeat/module/etcd/leader/_meta/docs.asciidoc

@@ -1 +1,2 @@
 This is the leader metricset of the module etcd.
+This metrics is being read from the Etcd V2 endpoint and won't show any activity regarding Etcd V3.

metricbeat/module/etcd/leader/leader.go

@@ -18,6 +18,7 @@
 package leader
 
 import (
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/metricbeat/helper"
 	"github.com/elastic/beats/metricbeat/mb"
@@ -27,6 +28,7 @@ import (
 const (
 	defaultScheme = "http"
 	defaultPath   = "/v2/stats/leader"
+	apiVersion    = "2"
 )
 
 var (
@@ -79,5 +81,6 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) {
 
 	reporter.Event(mb.Event{
 		MetricSetFields: eventMapping(content),
+		ModuleFields:    common.MapStr{"apiVersion": apiVersion},
 	})
 }