Cherry-pick #14794 to 7.x: [metricbeat]kubernetes kube-state-metrics service

CHANGELOG.next.asciidoc

@@ -527,6 +527,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add metrics to envoyproxy server metricset and support for envoy proxy 1.12. {pull}14416[14416] {issue}13642[13642]
 - Add module for ActiveMQ. {pull}14788[14788]
 - Enable wildcard for cloudwatch metricset namespace. {pull}14971[14971] {issue}14965[14965]
+- Add `kube-state-metrics` `state_service` metrics for kubernetes module. {pull}14794[14794]
 - Add usage metricset in aws modules. {pull}14925[14925] {issue}14935[14935]
 - Add billing metricset in aws modules. {pull}14801[14801] {issue}14934[14934]
 - Add AWS SNS metricset. {pull}14946[14946]

metricbeat/docs/fields.asciidoc

@@ -19476,6 +19476,103 @@ type: keyword
 
 --
 
+[float]
+=== service
+
+kubernetes service metrics
+
+
+
+*`kubernetes.service.name`*::
++
+--
+Service name.
+
+type: keyword
+
+--
+
+*`kubernetes.service.cluster_ip`*::
++
+--
+Internal IP for the service.
+
+type: ip
+
+--
+
+*`kubernetes.service.external_name`*::
++
+--
+Service external DNS name
+
+type: keyword
+
+--
+
+*`kubernetes.service.external_ip`*::
++
+--
+Service external IP
+
+type: keyword
+
+--
+
+*`kubernetes.service.load_balancer_ip`*::
++
+--
+Load Balancer service IP
+
+type: keyword
+
+--
+
+*`kubernetes.service.type`*::
++
+--
+Service type
+
+type: keyword
+
+--
+
+*`kubernetes.service.ingress_ip`*::
++
+--
+Ingress IP
+
+type: keyword
+
+--
+
+*`kubernetes.service.ingress_hostname`*::
++
+--
+Ingress Hostname
+
+type: ip
+
+--
+
+*`kubernetes.service.labels.*`*::
++
+--
+Labels for service
+
+type: object
+
+--
+
+*`kubernetes.service.created`*::
++
+--
+Service creation date
+
+type: date
+
+--
+
 [float]
 === statefulset
 

metricbeat/docs/modules/kubernetes.asciidoc

@@ -95,6 +95,7 @@ metricbeat.modules:
     - state_container
     - state_cronjob
     - state_resourcequota
+    - state_service
   period: 10s
   hosts: ["kube-state-metrics:8080"]
 
@@ -186,6 +187,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-kubernetes-state_resourcequota,state_resourcequota>>
 
+* <<metricbeat-metricset-kubernetes-state_service,state_service>>
+
 * <<metricbeat-metricset-kubernetes-state_statefulset,state_statefulset>>
 
 * <<metricbeat-metricset-kubernetes-system,system>>
@@ -222,6 +225,8 @@ include::kubernetes/state_replicaset.asciidoc[]
 
 include::kubernetes/state_resourcequota.asciidoc[]
 
+include::kubernetes/state_service.asciidoc[]
+
 include::kubernetes/state_statefulset.asciidoc[]
 
 include::kubernetes/system.asciidoc[]

metricbeat/docs/modules/kubernetes/state_service.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-kubernetes-state_service]]
+=== Kubernetes state_service metricset
+
+experimental[]
+
+include::../../../module/kubernetes/state_service/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-kubernetes,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/kubernetes/state_service/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -108,7 +108,7 @@ This file is generated! See scripts/mage/docs_collector.go
 .2+| .2+|  |<<metricbeat-metricset-kibana-stats,stats>>   
 |<<metricbeat-metricset-kibana-status,status>>   
 |<<metricbeat-module-kubernetes,Kubernetes>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.18+| .18+|  |<<metricbeat-metricset-kubernetes-apiserver,apiserver>>   
+.19+| .19+|  |<<metricbeat-metricset-kubernetes-apiserver,apiserver>>   
 |<<metricbeat-metricset-kubernetes-container,container>>   
 |<<metricbeat-metricset-kubernetes-controllermanager,controllermanager>> beta[]  
 |<<metricbeat-metricset-kubernetes-event,event>>   
@@ -123,6 +123,7 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-kubernetes-state_pod,state_pod>>   
 |<<metricbeat-metricset-kubernetes-state_replicaset,state_replicaset>>   
 |<<metricbeat-metricset-kubernetes-state_resourcequota,state_resourcequota>> beta[]  
+|<<metricbeat-metricset-kubernetes-state_service,state_service>> experimental[]  
 |<<metricbeat-metricset-kubernetes-state_statefulset,state_statefulset>>   
 |<<metricbeat-metricset-kubernetes-system,system>>   
 |<<metricbeat-metricset-kubernetes-volume,volume>>   

metricbeat/helper/prometheus/metric.go

@@ -21,6 +21,7 @@ import (
 	"math"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/elastic/beats/libbeat/common"
 
@@ -37,6 +38,38 @@ type MetricMap interface {
 
 	// GetValue returns the resulting value
 	GetValue(m *dto.Metric) interface{}
+
+	// GetConfiguration returns the configuration for the metric
+	GetConfiguration() Configuration
+}
+
+// Configuration for mappings that needs extended treatment
+type Configuration struct {
+	// StoreNonMappedLables indicates if labels found at the metric that are
+	// not found at the label map should be part of the resulting event.
+	// This setting should be used when the label name is not known beforehand
+	StoreNonMappedLabels bool
+	// NonMappedLabelsPlacement is used when StoreNonMappedLabels is set to true, and
+	// defines the key path at the event under which to store the dynamically found labels.
+	// This key path will be added to the events that match this metric along with a subset of
+	// key/value pairs will be created under it, one for each non mapped label found.
+	//
+	// Example:
+	//
+	// given a metric family in a prometheus resource in the form:
+	// 		metric1{label1="value1",label2="value2"} 1
+	// and not mapping labels but using this entry on a the MetriMap definition:
+	// 		"metric1": ExtendedInfoMetric(Configuration{StoreNonMappedLabels: true, NonMappedLabelsPlacement: "mypath"}),
+	// would output an event that contains a metricset field as follows
+	// 		"mypath": {"label1":"value1","label2":"value2"}
+	//
+	NonMappedLabelsPlacement string
+	// MetricProcessing options are a set of functions that will be
+	// applied to metrics after they are retrieved
+	MetricProcessingOptions []MetricOption
+	// ExtraFields is used to add fields to the
+	// event where this metric is included
+	ExtraFields common.MapStr
 }
 
 // MetricOption adds settings to Metric objects behavior
@@ -57,6 +90,11 @@ func OpLowercaseValue() MetricOption {
 	return opLowercaseValue{}
 }
 
+// OpUnixTimestampValue parses a value into a Unix timestamp
+func OpUnixTimestampValue() MetricOption {
+	return opUnixTimestampValue{}
+}
+
 // OpMultiplyBuckets multiplies bucket labels in histograms, useful to change units
 func OpMultiplyBuckets(multiplier float64) MetricOption {
 	return opMultiplyBuckets{
@@ -67,8 +105,8 @@ func OpMultiplyBuckets(multiplier float64) MetricOption {
 // Metric directly maps a Prometheus metric to a Metricbeat field
 func Metric(field string, options ...MetricOption) MetricMap {
 	return &commonMetric{
-		field:   field,
-		options: options,
+		field:  field,
+		config: Configuration{MetricProcessingOptions: options},
 	}
 }
 
@@ -77,8 +115,8 @@ func Metric(field string, options ...MetricOption) MetricMap {
 func KeywordMetric(field, keyword string, options ...MetricOption) MetricMap {
 	return &keywordMetric{
 		commonMetric{
-			field:   field,
-			options: options,
+			field:  field,
+			config: Configuration{MetricProcessingOptions: options},
 		},
 		keyword,
 	}
@@ -88,8 +126,8 @@ func KeywordMetric(field, keyword string, options ...MetricOption) MetricMap {
 func BooleanMetric(field string, options ...MetricOption) MetricMap {
 	return &booleanMetric{
 		commonMetric{
-			field:   field,
-			options: options,
+			field:  field,
+			config: Configuration{MetricProcessingOptions: options},
 		},
 	}
 }
@@ -99,8 +137,8 @@ func BooleanMetric(field string, options ...MetricOption) MetricMap {
 func LabelMetric(field, label string, options ...MetricOption) MetricMap {
 	return &labelMetric{
 		commonMetric{
-			field:   field,
-			options: options,
+			field:  field,
+			config: Configuration{MetricProcessingOptions: options},
 		},
 		label,
 	}
@@ -111,26 +149,50 @@ func LabelMetric(field, label string, options ...MetricOption) MetricMap {
 func InfoMetric(options ...MetricOption) MetricMap {
 	return &infoMetric{
 		commonMetric{
-			options: options,
+			config: Configuration{MetricProcessingOptions: options},
+		},
+	}
+}
+
+// ExtendedInfoMetric obtains info labels from the given metric and puts them
+// into events matching all the key labels present in the metric
+func ExtendedInfoMetric(configuration Configuration) MetricMap {
+	return &infoMetric{
+		commonMetric{
+			config: configuration,
 		},
 	}
 }
 
+// ExtendedMetric is a metric item that allows extended behaviour
+// through configuration
+func ExtendedMetric(field string, configuration Configuration) MetricMap {
+	return &commonMetric{
+		field:  field,
+		config: configuration,
+	}
+}
+
 type commonMetric struct {
-	field   string
-	options []MetricOption
+	field  string
+	config Configuration
 }
 
 // GetOptions returns the list of metric options
 func (m *commonMetric) GetOptions() []MetricOption {
-	return m.options
+	return m.config.MetricProcessingOptions
 }
 
 // GetField returns the resulting field name
 func (m *commonMetric) GetField() string {
 	return m.field
 }
 
+// GetConfiguration returns the configuration for the metric
+func (m *commonMetric) GetConfiguration() Configuration {
+	return m.config
+}
+
 // GetValue returns the resulting value
 func (m *commonMetric) GetValue(metric *dto.Metric) interface{} {
 	counter := metric.GetCounter()
@@ -315,3 +377,11 @@ func (o opMultiplyBuckets) Process(field string, value interface{}, labels commo
 	histogram["sum"] = sum * o.multiplier
 	return field, histogram, labels
 }
+
+type opUnixTimestampValue struct {
+}
+
+// Process converts a value in seconds into an unix time
+func (o opUnixTimestampValue) Process(field string, value interface{}, labels common.MapStr) (string, interface{}, common.MapStr) {
+	return field, common.Time(time.Unix(int64(value.(float64)), 0)), labels
+}