[metricbeat]kubernetes kube-state-metrics service

metricbeat/docs/fields.asciidoc

@@ -19318,6 +19318,49 @@ type: keyword
 
 --
 
+[float]
+=== service
+
+kubernetes service metrics
+
+
+
+*`kubernetes.service.name`*::
++
+--
+Service name.
+
+type: keyword
+
+--
+
+*`kubernetes.service.cluster_ip`*::
++
+--
+Internal IP for the service.
+
+type: ip
+
+--
+
+*`kubernetes.service.external_name`*::
++
+--
+Service external DNS name
+
+type: keyword
+
+--
+
+*`kubernetes.service.load_balancer_ip`*::
++
+--
+Load Balancer service IP name
+
+type: keyword
+
+--
+
 [float]
 === statefulset
 

metricbeat/docs/modules/kubernetes.asciidoc

@@ -186,6 +186,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-kubernetes-state_resourcequota,state_resourcequota>>
 
+* <<metricbeat-metricset-kubernetes-state_service,state_service>>
+
 * <<metricbeat-metricset-kubernetes-state_statefulset,state_statefulset>>
 
 * <<metricbeat-metricset-kubernetes-system,system>>
@@ -222,6 +224,8 @@ include::kubernetes/state_replicaset.asciidoc[]
 
 include::kubernetes/state_resourcequota.asciidoc[]
 
+include::kubernetes/state_service.asciidoc[]
+
 include::kubernetes/state_statefulset.asciidoc[]
 
 include::kubernetes/system.asciidoc[]

metricbeat/docs/modules/kubernetes/state_service.asciidoc

@@ -0,0 +1,21 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-kubernetes-state_service]]
+=== Kubernetes state_service metricset
+
+include::../../../module/kubernetes/state_service/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-kubernetes,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/kubernetes/state_service/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -105,7 +105,7 @@ This file is generated! See scripts/mage/docs_collector.go
 .2+| .2+|  |<<metricbeat-metricset-kibana-stats,stats>>   
 |<<metricbeat-metricset-kibana-status,status>>   
 |<<metricbeat-module-kubernetes,Kubernetes>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.18+| .18+|  |<<metricbeat-metricset-kubernetes-apiserver,apiserver>>   
+.19+| .19+|  |<<metricbeat-metricset-kubernetes-apiserver,apiserver>>   
 |<<metricbeat-metricset-kubernetes-container,container>>   
 |<<metricbeat-metricset-kubernetes-controllermanager,controllermanager>>   
 |<<metricbeat-metricset-kubernetes-event,event>>   
@@ -120,6 +120,7 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-kubernetes-state_pod,state_pod>>   
 |<<metricbeat-metricset-kubernetes-state_replicaset,state_replicaset>>   
 |<<metricbeat-metricset-kubernetes-state_resourcequota,state_resourcequota>>   
+|<<metricbeat-metricset-kubernetes-state_service,state_service>>   
 |<<metricbeat-metricset-kubernetes-state_statefulset,state_statefulset>>   
 |<<metricbeat-metricset-kubernetes-system,system>>   
 |<<metricbeat-metricset-kubernetes-volume,volume>>   

metricbeat/helper/prometheus/metric.go

@@ -21,6 +21,7 @@ import (
 	"math"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/elastic/beats/libbeat/common"
 
@@ -37,6 +38,25 @@ type MetricMap interface {
 
 	// GetValue returns the resulting value
 	GetValue(m *dto.Metric) interface{}
+
+	// GetConfiguration returns the configuration for the metric
+	GetConfiguration() Configuration
+}
+
+// Configuration for mappings that needs extended treatment
+type Configuration struct {
+	// StoreNonMappedLables indicates if labels found at the metric that are
+	// not found at the label map should be part of the resulting event
+	StoreNonMappedLabels bool
+	// NonMappedLabelsPlacement is used when StoreNonMappedLabels is set to true, and
+	// defines the key at the event to store labels
+	NonMappedLabelsPlacement string
+	// MetricProcessing options are a set of functions that will be
+	// applied to metrics after they are retrieved
+	MetricProcessingOptions []MetricOption
+	// ExtraFields is used to add fields to the
+	// event where this metric is included
+	ExtraFields common.MapStr
 }
 
 // MetricOption adds settings to Metric objects behavior
@@ -57,6 +77,11 @@ func OpLowercaseValue() MetricOption {
 	return opLowercaseValue{}
 }
 
+// OpUnixTimestampValue parses a value into a Unix timestamp
+func OpUnixTimestampValue() MetricOption {
+	return opUnixTimestampValue{}
+}
+
 // OpMultiplyBuckets multiplies bucket labels in histograms, useful to change units
 func OpMultiplyBuckets(multiplier float64) MetricOption {
 	return opMultiplyBuckets{
@@ -67,8 +92,8 @@ func OpMultiplyBuckets(multiplier float64) MetricOption {
 // Metric directly maps a Prometheus metric to a Metricbeat field
 func Metric(field string, options ...MetricOption) MetricMap {
 	return &commonMetric{
-		field:   field,
-		options: options,
+		field:  field,
+		config: Configuration{MetricProcessingOptions: options},
 	}
 }
 
@@ -77,8 +102,8 @@ func Metric(field string, options ...MetricOption) MetricMap {
 func KeywordMetric(field, keyword string, options ...MetricOption) MetricMap {
 	return &keywordMetric{
 		commonMetric{
-			field:   field,
-			options: options,
+			field:  field,
+			config: Configuration{MetricProcessingOptions: options},
 		},
 		keyword,
 	}
@@ -88,8 +113,8 @@ func KeywordMetric(field, keyword string, options ...MetricOption) MetricMap {
 func BooleanMetric(field string, options ...MetricOption) MetricMap {
 	return &booleanMetric{
 		commonMetric{
-			field:   field,
-			options: options,
+			field:  field,
+			config: Configuration{MetricProcessingOptions: options},
 		},
 	}
 }
@@ -99,8 +124,8 @@ func BooleanMetric(field string, options ...MetricOption) MetricMap {
 func LabelMetric(field, label string, options ...MetricOption) MetricMap {
 	return &labelMetric{
 		commonMetric{
-			field:   field,
-			options: options,
+			field:  field,
+			config: Configuration{MetricProcessingOptions: options},
 		},
 		label,
 	}
@@ -111,26 +136,50 @@ func LabelMetric(field, label string, options ...MetricOption) MetricMap {
 func InfoMetric(options ...MetricOption) MetricMap {
 	return &infoMetric{
 		commonMetric{
-			options: options,
+			config: Configuration{MetricProcessingOptions: options},
+		},
+	}
+}
+
+// ExtendedInfoMetric obtains info labels from the given metric and puts them
+// into events matching all the key labels present in the metric
+func ExtendedInfoMetric(configuration Configuration) MetricMap {
+	return &infoMetric{
+		commonMetric{
+			config: configuration,
 		},
 	}
 }
 
+// ExtendedMetric is a metric item that allows extended behaviour
+// through configuration
+func ExtendedMetric(field string, configuration Configuration) MetricMap {
+	return &commonMetric{
+		field:  field,
+		config: configuration,
+	}
+}
+
 type commonMetric struct {
-	field   string
-	options []MetricOption
+	field  string
+	config Configuration
 }
 
 // GetOptions returns the list of metric options
 func (m *commonMetric) GetOptions() []MetricOption {
-	return m.options
+	return m.config.MetricProcessingOptions
 }
 
 // GetField returns the resulting field name
 func (m *commonMetric) GetField() string {
 	return m.field
 }
 
+// GetConfiguration returns the configuration for the metric
+func (m *commonMetric) GetConfiguration() Configuration {
+	return m.config
+}
+
 // GetValue returns the resulting value
 func (m *commonMetric) GetValue(metric *dto.Metric) interface{} {
 	counter := metric.GetCounter()
@@ -315,3 +364,11 @@ func (o opMultiplyBuckets) Process(field string, value interface{}, labels commo
 	histogram["sum"] = sum * o.multiplier
 	return field, histogram, labels
 }
+
+type opUnixTimestampValue struct {
+}
+
+// Process converts a value in seconds into an unix time
+func (o opUnixTimestampValue) Process(field string, value interface{}, labels common.MapStr) (string, interface{}, common.MapStr) {
+	return field, time.Unix(int64(value.(float64)), 0), labels
+}

metricbeat/helper/prometheus/prometheus.go

@@ -119,6 +119,9 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS
 	for _, family := range families {
 		for _, metric := range family.GetMetric() {
 			m, ok := mapping.Metrics[family.GetName()]
+			if m == nil {
+				continue
+			}
 
 			// Ignore unknown metrics
 			if !ok {
@@ -133,6 +136,16 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS
 				continue
 			}
 
+			storeAllLabels := false
+			labelsLocation := ""
+			var extraFields common.MapStr
+			if m != nil {
+				c := m.GetConfiguration()
+				storeAllLabels = c.StoreNonMappedLabels
+				labelsLocation = c.NonMappedLabelsPlacement
+				extraFields = c.ExtraFields
+			}
+
 			// Apply extra options
 			allLabels := getLabels(metric)
 			for _, option := range m.GetOptions() {
@@ -142,16 +155,31 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS
 			// Convert labels
 			labels := common.MapStr{}
 			keyLabels := common.MapStr{}
+
 			for k, v := range allLabels {
 				if l, ok := mapping.Labels[k]; ok {
 					if l.IsKey() {
 						keyLabels.Put(l.GetField(), v)
 					} else {
 						labels.Put(l.GetField(), v)
 					}
+				} else if storeAllLabels {
+					// if label for this metric is not found at the label mappings but
+					// it is configured to store any labels found, make it so
+					// TODO dedot
+					labels.Put(labelsLocation+"."+k, v)
 				}
 			}
 
+			// if extra fields have been added through metric configuration
+			// add them to labels.
+			//
+			// not considering these extra fields to be keylabels as that case
+			// have not appeared yet
+			for k, v := range extraFields {
+				labels.Put(k, v)
+			}
+
 			// Keep a info document if it's an infoMetric
 			if _, ok = m.(*infoMetric); ok {
 				labels.DeepUpdate(keyLabels)
@@ -230,7 +258,7 @@ func (p *prometheus) ReportProcessedMetrics(mapping *MetricsMapping, r mb.Report
 }
 
 func getEvent(m map[string]common.MapStr, labels common.MapStr) common.MapStr {
-	hash := LabelHash(labels.Flatten())
+	hash := labels.String()
 	res, ok := m[hash]
 	if !ok {
 		res = labels