From b15860e5ca5f74d1eb93fe7af47f18d170f0688a Mon Sep 17 00:00:00 2001 From: Pablo Mercado Date: Fri, 29 Nov 2019 12:04:07 +0100 Subject: [PATCH 01/10] initial commit for PV metricset --- metricbeat/helper/prometheus/metric.go | 81 ++++++++++++++++--- metricbeat/helper/prometheus/prometheus.go | 30 ++++++- .../helper/prometheus/prometheus_test.go | 71 ++++++++++++++++ .../state_persisentvolume/README.md | 18 +++++ .../state_persisentvolume/_meta/fields.yml | 25 ++++++ 5 files changed, 212 insertions(+), 13 deletions(-) create mode 100644 metricbeat/module/kubernetes/state_persisentvolume/README.md create mode 100644 metricbeat/module/kubernetes/state_persisentvolume/_meta/fields.yml diff --git a/metricbeat/helper/prometheus/metric.go b/metricbeat/helper/prometheus/metric.go index 0fb1df2ba990..f1a250edb982 100644 --- a/metricbeat/helper/prometheus/metric.go +++ b/metricbeat/helper/prometheus/metric.go @@ -21,6 +21,7 @@ import ( "math" "strconv" "strings" + "time" "github.com/elastic/beats/libbeat/common" @@ -37,6 +38,25 @@ type MetricMap interface { // GetValue returns the resulting value GetValue(m *dto.Metric) interface{} + + // GetConfiguration returns the configuration for the metric + GetConfiguration() Configuration +} + +// Configuration for mappings that needs extended treatment +type Configuration struct { + // StoreNonMappedLables indicates if labels found at the metric that are + // not found at the label map should be part of the resulting event + StoreNonMappedLabels bool + // NonMappedLabelsPlacement is used when StoreNonMappedLabels is set to true, and + // defines the key at the event to store labels + NonMappedLabelsPlacement string + // MetricProcessing options are a set of functions that will be + // applied to metrics after they are retrieved + MetricProcessingOptions []MetricOption + // ExtraFields is used to add fields to the + // event where this metric is included + ExtraFields common.MapStr } // MetricOption adds settings to Metric objects behavior @@ -57,6 +77,11 @@ func OpLowercaseValue() MetricOption { return opLowercaseValue{} } +// OpUnixTimestampValue parses a value into a Unix timestamp +func OpUnixTimestampValue() MetricOption { + return opUnixTimestampValue{} +} + // OpMultiplyBuckets multiplies bucket labels in histograms, useful to change units func OpMultiplyBuckets(multiplier float64) MetricOption { return opMultiplyBuckets{ @@ -67,8 +92,8 @@ func OpMultiplyBuckets(multiplier float64) MetricOption { // Metric directly maps a Prometheus metric to a Metricbeat field func Metric(field string, options ...MetricOption) MetricMap { return &commonMetric{ - field: field, - options: options, + field: field, + config: Configuration{MetricProcessingOptions: options}, } } @@ -77,8 +102,8 @@ func Metric(field string, options ...MetricOption) MetricMap { func KeywordMetric(field, keyword string, options ...MetricOption) MetricMap { return &keywordMetric{ commonMetric{ - field: field, - options: options, + field: field, + config: Configuration{MetricProcessingOptions: options}, }, keyword, } @@ -88,8 +113,8 @@ func KeywordMetric(field, keyword string, options ...MetricOption) MetricMap { func BooleanMetric(field string, options ...MetricOption) MetricMap { return &booleanMetric{ commonMetric{ - field: field, - options: options, + field: field, + config: Configuration{MetricProcessingOptions: options}, }, } } @@ -99,8 +124,8 @@ func BooleanMetric(field string, options ...MetricOption) MetricMap { func LabelMetric(field, label string, options ...MetricOption) MetricMap { return &labelMetric{ commonMetric{ - field: field, - options: options, + field: field, + config: Configuration{MetricProcessingOptions: options}, }, label, } @@ -111,19 +136,38 @@ func LabelMetric(field, label string, options ...MetricOption) MetricMap { func InfoMetric(options ...MetricOption) MetricMap { return &infoMetric{ commonMetric{ - options: options, + config: Configuration{MetricProcessingOptions: options}, + }, + } +} + +// ExtendedInfoMetric obtains info labels from the given metric and puts them +// into events matching all the key labels present in the metric +func ExtendedInfoMetric(configuration Configuration) MetricMap { + return &infoMetric{ + commonMetric{ + config: configuration, }, } } +// ExtendedMetric is a metric item that allows extended behaviour +// through configuration +func ExtendedMetric(field string, configuration Configuration) MetricMap { + return &commonMetric{ + field: field, + config: configuration, + } +} + type commonMetric struct { - field string - options []MetricOption + field string + config Configuration } // GetOptions returns the list of metric options func (m *commonMetric) GetOptions() []MetricOption { - return m.options + return m.config.MetricProcessingOptions } // GetField returns the resulting field name @@ -131,6 +175,11 @@ func (m *commonMetric) GetField() string { return m.field } +// GetConfiguration returns the configuration for the metric +func (m *commonMetric) GetConfiguration() Configuration { + return m.config +} + // GetValue returns the resulting value func (m *commonMetric) GetValue(metric *dto.Metric) interface{} { counter := metric.GetCounter() @@ -315,3 +364,11 @@ func (o opMultiplyBuckets) Process(field string, value interface{}, labels commo histogram["sum"] = sum * o.multiplier return field, histogram, labels } + +type opUnixTimestampValue struct { +} + +// Process converts a value in seconds into an unix time +func (o opUnixTimestampValue) Process(field string, value interface{}, labels common.MapStr) (string, interface{}, common.MapStr) { + return field, common.Time(time.Unix(int64(value.(float64)), 0)), labels +} diff --git a/metricbeat/helper/prometheus/prometheus.go b/metricbeat/helper/prometheus/prometheus.go index a60838c7d82a..acbbcdaa422d 100644 --- a/metricbeat/helper/prometheus/prometheus.go +++ b/metricbeat/helper/prometheus/prometheus.go @@ -119,6 +119,9 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS for _, family := range families { for _, metric := range family.GetMetric() { m, ok := mapping.Metrics[family.GetName()] + if m == nil { + continue + } // Ignore unknown metrics if !ok { @@ -133,6 +136,16 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS continue } + storeAllLabels := false + labelsLocation := "" + var extraFields common.MapStr + if m != nil { + c := m.GetConfiguration() + storeAllLabels = c.StoreNonMappedLabels + labelsLocation = c.NonMappedLabelsPlacement + extraFields = c.ExtraFields + } + // Apply extra options allLabels := getLabels(metric) for _, option := range m.GetOptions() { @@ -142,6 +155,7 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS // Convert labels labels := common.MapStr{} keyLabels := common.MapStr{} + for k, v := range allLabels { if l, ok := mapping.Labels[k]; ok { if l.IsKey() { @@ -149,9 +163,23 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS } else { labels.Put(l.GetField(), v) } + } else if storeAllLabels { + // if label for this metric is not found at the label mappings but + // it is configured to store any labels found, make it so + // TODO dedot + labels.Put(labelsLocation+"."+k, v) } } + // if extra fields have been added through metric configuration + // add them to labels. + // + // not considering these extra fields to be keylabels as that case + // have not appeared yet + for k, v := range extraFields { + labels.Put(k, v) + } + // Keep a info document if it's an infoMetric if _, ok = m.(*infoMetric); ok { labels.DeepUpdate(keyLabels) @@ -230,7 +258,7 @@ func (p *prometheus) ReportProcessedMetrics(mapping *MetricsMapping, r mb.Report } func getEvent(m map[string]common.MapStr, labels common.MapStr) common.MapStr { - hash := LabelHash(labels.Flatten()) + hash := labels.String() res, ok := m[hash] if !ok { res = labels diff --git a/metricbeat/helper/prometheus/prometheus_test.go b/metricbeat/helper/prometheus/prometheus_test.go index dbe1ca92260b..a3ec53443520 100644 --- a/metricbeat/helper/prometheus/prometheus_test.go +++ b/metricbeat/helper/prometheus/prometheus_test.go @@ -163,6 +163,15 @@ metrics_force_propagation_ms{kind="jedi",quantile="1"} 30 metrics_force_propagation_ms_sum{kind="jedi"} 50 metrics_force_propagation_ms_count{kind="jedi"} 651 +` + + promGaugeLabeled = ` +# TYPE metrics_that_inform_labels gauge +metrics_that_inform_labels{label1="I am 1", label2="I am 2"} 1 +metrics_that_inform_labels{label1="I am 1", label3="I am 3"} 1 +# TYPE metrics_that_use_labels gauge +metrics_that_use_labels{label1="I am 1"} 20 + ` ) @@ -858,6 +867,68 @@ func TestPrometheusKeyLabels(t *testing.T) { }, }, }, + + { + testName: "Test gauge InfoMetrics using ExtendedInfoMetric", + prometheusResponse: promGaugeLabeled, + mapping: &MetricsMapping{ + Metrics: map[string]MetricMap{ + "metrics_that_inform_labels": ExtendedInfoMetric(Configuration{StoreNonMappedLabels: true, NonMappedLabelsPlacement: "metrics.other_labels"}), + "metrics_that_use_labels": Metric("metrics.value"), + }, + Labels: map[string]LabelMap{ + "label1": KeyLabel("metrics.label1"), + }, + }, + expectedEvents: []common.MapStr{ + common.MapStr{ + "metrics": common.MapStr{ + "value": 20.0, + "label1": "I am 1", + "other_labels": common.MapStr{ + "label2": "I am 2", + "label3": "I am 3", + }, + }, + }, + }, + }, + + { + testName: "Test gauge InfoMetrics using ExtendedInfoMetric and extra fields", + prometheusResponse: promGaugeLabeled, + mapping: &MetricsMapping{ + Metrics: map[string]MetricMap{ + "metrics_that_inform_labels": ExtendedInfoMetric(Configuration{ + StoreNonMappedLabels: true, + NonMappedLabelsPlacement: "metrics.other_labels", + ExtraFields: common.MapStr{ + "metrics.extra.field1": "extra1", + "metrics.extra.field2": "extra2", + }}), + "metrics_that_use_labels": Metric("metrics.value"), + }, + Labels: map[string]LabelMap{ + "label1": KeyLabel("metrics.label1"), + }, + }, + expectedEvents: []common.MapStr{ + common.MapStr{ + "metrics": common.MapStr{ + "value": 20.0, + "label1": "I am 1", + "other_labels": common.MapStr{ + "label2": "I am 2", + "label3": "I am 3", + }, + "extra": common.MapStr{ + "field1": "extra1", + "field2": "extra2", + }, + }, + }, + }, + }, } for _, tc := range testCases { diff --git a/metricbeat/module/kubernetes/state_persisentvolume/README.md b/metricbeat/module/kubernetes/state_persisentvolume/README.md new file mode 100644 index 000000000000..5817c0cd442d --- /dev/null +++ b/metricbeat/module/kubernetes/state_persisentvolume/README.md @@ -0,0 +1,18 @@ +# Kube-state-metrics/PersistentVolume + + +This metricset connects to kube-state-metrics endpoint to retrieve and report Persistent Volume metrics. + +## Version history + +- November 2019, first release using kube-state-metrics `v1.8.0`. + +## Configuration + +See the metricset documentation for the configuration reference. + +## Manual testing + +// TODO + +Then run metricbeat pointing to the kube-state-metrics endpoint. diff --git a/metricbeat/module/kubernetes/state_persisentvolume/_meta/fields.yml b/metricbeat/module/kubernetes/state_persisentvolume/_meta/fields.yml new file mode 100644 index 000000000000..0db6d711bbc2 --- /dev/null +++ b/metricbeat/module/kubernetes/state_persisentvolume/_meta/fields.yml @@ -0,0 +1,25 @@ +- name: persistentvolume + type: group + description: > + kubernetes persistent volume metrics from kube-state-metrics + release: experimental + fields: + - name: name + type: keyword + description: Volume name. + - name: capacity.bytes + type: long + description: Volume capacity + - name: phase + type: keyword + description: Volume phase according to kubernetes + - name: storage_class + type: keyword + description: Storage class for the volume + - name: labels.* + type: object + object_type: keyword + description: Labels for volume + + + From 8eb89dfabf67bf350eff9d291b9b99b90b1e9f11 Mon Sep 17 00:00:00 2001 From: Pablo Mercado Date: Fri, 29 Nov 2019 13:13:09 +0100 Subject: [PATCH 02/10] flesh out metricset --- metricbeat/docs/fields.asciidoc | 52 ++++++++++++ metricbeat/include/list_docker.go | 1 + metricbeat/module/kubernetes/fields.go | 2 +- .../README.md | 0 .../_meta/fields.yml | 0 .../_meta/test/ksm-unit-1.8 | 39 +++++++++ .../_meta/test/ksm-unit-1.8.expected | 1 + .../state_persistentvolume.go | 84 +++++++++++++++++++ .../state_persistentvolume_test.go | 39 +++++++++ 9 files changed, 217 insertions(+), 1 deletion(-) rename metricbeat/module/kubernetes/{state_persisentvolume => state_persistentvolume}/README.md (100%) rename metricbeat/module/kubernetes/{state_persisentvolume => state_persistentvolume}/_meta/fields.yml (100%) create mode 100644 metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8 create mode 100644 metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8.expected create mode 100644 metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go create mode 100644 metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume_test.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index be4848bb7a68..a1f7d497acb8 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -19267,6 +19267,58 @@ type: long -- +[float] +=== persistentvolume + +kubernetes persistent volume metrics from kube-state-metrics + + + +*`kubernetes.persistentvolume.name`*:: ++ +-- +Volume name. + +type: keyword + +-- + +*`kubernetes.persistentvolume.capacity.bytes`*:: ++ +-- +Volume capacity + +type: long + +-- + +*`kubernetes.persistentvolume.phase`*:: ++ +-- +Volume phase according to kubernetes + +type: keyword + +-- + +*`kubernetes.persistentvolume.storage_class`*:: ++ +-- +Storage class for the volume + +type: keyword + +-- + +*`kubernetes.persistentvolume.labels.*`*:: ++ +-- +Labels for volume + +type: object + +-- + [float] === pod diff --git a/metricbeat/include/list_docker.go b/metricbeat/include/list_docker.go index 5a61d9a92535..39e662554c5a 100644 --- a/metricbeat/include/list_docker.go +++ b/metricbeat/include/list_docker.go @@ -44,6 +44,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_cronjob" _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_deployment" _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_node" + _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_persistentvolume" _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_pod" _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_replicaset" _ "github.com/elastic/beats/metricbeat/module/kubernetes/state_resourcequota" diff --git a/metricbeat/module/kubernetes/fields.go b/metricbeat/module/kubernetes/fields.go index 3c8f08b3a720..4762ac02c50c 100644 --- a/metricbeat/module/kubernetes/fields.go +++ b/metricbeat/module/kubernetes/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKubernetes returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/kubernetes. func AssetKubernetes() string { - return "eJzsXU9z5KiSv/enIPrk2fCrw8bGHnzYiPc878V29Ew/r91+c9jYqMFSVhVjCdSA7K759Bugf5QECJWostuWDg7Xv8wfmUmSJJD8BT3C/go9lg/AKUgQHxCSRGZwhT5+bt/8+AGhFETCSSEJo1fovz4ghFD3BZSD5CRRv+aQARZwhbb4A0ICpCR0K67Q/34UIvt4iT7upCw+/p/6bMe4XCeMbsj2Cm1wJuADQhsCWSquNIO/IIpz6MFTj9wXigNnZVG/Y4Gnnk90w3iO1dsI0xQJiSURkiQCsQ0qWCpQjineQooe9gafVU3BRGMiwgURwJ+At5/YQHmA9eT315tPqCJoiLJ5DkXaPH1oJjwO30oQcpVkBKg8+EqD8xH2z4ynvc88aNVzrekh+A5JqfTaMBJeFBwEK3kC8XDcVpQhRVbafQCifDglBhf5AYyEFfEBIE0WXSRZKSTwS81UFDiBy1Y6P3lxPQF/iAfrv79+vUEDkgPLZGlEUWieA5JDnlQClWvFKL4aagyaBRqw6GNJ+X7NSxoPxm8gd8CR3EHDA5UCBEr5HvUZ9cE8EtrnNgPJZ0JT5V1r6iMqyQtG4/qohiTaYZpmyksZQvGi6fvumUiUU9ck0YY1mglwE0/ABWERTaMm2KIYNrMPQUvuYHCbCaHpJDbCfeY5yB2LaI+6Y1qIDhrNREQzbFvcp9qwLThLQAgrR5sh2sZ7k15SlCsByeDzhmbKyoes7/cGDbm+uUcCEkbTPrKOUw4543s1rJMUqFw97LvIbMg3Y3Rr+bCKy66Q68cHqP6mvoQIRQ3PGsMYxCfCZYmzcyKsWY4B3KRixQqgq4SVA+83Cu2A9ZcyfwCuPK4iiDYkg/YLjLvVKCTmEtIIRnNXGQwShCagXUxt3A0PawdQE4Fo1t+OqyXX0f6qFKsCeAJUkgxW/+ZsIXv4AxKbAqoP1lPk0PT5BgTKScJZ3Z1QB8etE1szRJnP1I8fV1LmZYYleQJkY+WDNt94G2iakh6hGvqjQAT5E6qeHVPTU0ArBJPUakD2aTWGQzrAOFHFBsxTaFiR92AQBaMCXlS9FYQp+h2CPr2CTZTBGh4CjaHiGoqd1DDoj29TTcOsI02VBln5+Dt5O4baJvGBsECWLEuvyfGCvIjRgjV3YzLLsASa7I+xZJu2REPwUpmoQlC9JlXgZI5Jo5DimVCLiU4XzEOZPII865BTs0Y7IiTbcpyjCoQbbGgoMQVFQ7PSZKjyThM5dFioGQhXb4aBeQE9dqjDNZmUnCs/Nl92n+gmI9udDDB1Rre8pJTQbdSpSuc/Ez1oqV+jmpE/qwwySVeV3KN48i7pX2tTICw1Fyt7XKZEruDJpYip7DU9pOnZ21sx5KCgQRqRZ0Oyz7wba6jEhM5b4zCk29KLssShZ5ZrSXJ7KjfFsv/BSMLmThFEA4JGeiV4FB/LUN7co1LgLVgE4Wq2CUX/1tkPbYB8VA8aybiN8DjxMQYmE4tT7rNx+JLmGZGv+Vy3Rqekfs041KKnmDoHrAO0mDIlFhfoUcCBYCujgHSEYQuLpbAqrGNSh0okOIN0vckYdn2xmXLUs5wYbVDSxQLhhqZ6zTY6LSSZxJnGjnCWsQRL/JCB+p23sRnJifzxWpvChlBIK/ht9r1zgxfqHadEENmgkurfQmpfwMvYNjx/PNKqX9hWheEbNtEZ4SdMMmxPQs13SK6ZMArpeWPTaRSuay2dtqkowQVOiNyr0NdOvfWo9TffvnQqSw6XjHJ2b18q2qWHC4UoT+BeqZg3ttujdxRxEPuqbaDrJ87mGAshHPwhRyxUilEIIIddxgekTcMC6HANK1rq6H046r4FjizDnS6Ufl0CqcTgbO4rjyt/NdBPDC0d+kevProMafOMALM2CHeMaUqID7YpoDfVR27v7vw9pAH8zPgjoVsB7jTYW5DHb1UzkQAZJpcCb2GDy8ySSJySHrQj6vJWig1y8GlHTfwH42fCo3k5UbW9hzG5ibjP5z3MKG4Zk3oni9gLCfnkycX7CHbsUjLD7/c+B7NLqI68X24udoY5xr1ldmFm9jnLMuDV4YdZGf7rllh9lCJOfv9FtqCec1f6ube5nnl7q/obj90XnEPYLuo/GY3I9xPdcCwkLxNZchgSXzbzVs1ZNvMum3mXzbwBzVg289qBLJt5gzEum3mXzbzLZt75m3ktUebU7b3PjD9+K6G0R5zHDH0KNKiAs9pyN384/6Ui2O6tqwdzXyxR0g2hROyihBP3LbEQ1jhNY9jwb41eFMERQ06hkLuoPDXF0e4jOYnSXzu+5g5mTd0+MWMprBI1YU8ks8+vjzFceCKJjiRixsB62aKh7DPYHeBM7mLsC++Yt1SRPRF0ij35fk4VHsdSVTi7m4OFJHcjW58EOAW+ImKdYyEdOZkHxjLA/UBv7ND6rju1rnVNBOrx+NBHo3erfuizn5Cw+roDs/RGtfu1yVmBGod032g/kTssEeaAtkCBY1nVCmn2Ctd+9YADoWpiq4T7uV+5BE1IhrkNzKFrr7Svq+FVcUEcEsZTUcm9NT5JcqjeKzCXJCkzzCshoB0WiCV6A3pqQah/KXFeWFAOnYkv7bchXMh1zYo66nVM39z7tQGo2ql5oI6Heq9vVeZhj5MDUixG8HS5EDFYiaswSPguw63h14pObQmQdsUByBNQizgSVuzXktkQdGMaFr2pnjv15kV3qymFgmutsF9040juX/dFu8Tu55iDxCmWOMjsR/RRUUJYCJYQ7WWeidx5deLrSPYuOX2Eb50QB9xP/iBfBwhYqTjoBJoBYdQv+ZMmmGvOfp66tE5cxpokIhQ970iyq13uMxbdiGNF0+TB19ErhvyrrhhiCsSfdi9JxKWMe0q+lYB0cphsiAoQmAHEkhxo06CQbdYZoY8Rwdz+gjgUHIRCU1eTcTkEQp9Y9gTp2oLxVH6h4WmTi89D4ILEt5y/3nxq683U1uNRV9zCQ4r3Y118aIRxXOdBDefhYXq6/tpQniD6uB32/tPPI7zN6eec6N04UKZnDMtZsuUsmeOJfZbsi7K3H/sY2bK33PYse8t7T7y95csW4h7gZQuxHfiyhdizhZiCVHYTzV/z72/a+G4hAfKk87QuWm02mXPbelQg5lA831182mzN21bIV46pyImUr0cnX606adPQy3796gmU5j+WrfoTBbTs0u+egXDewwZ9Y6HZcRC4D+ocJ7g7VK/j7HaHx3V+u41pSurM4Bzjt0muIsATncV3jwnjDMaYoMAejkJTJCE9HU1LpXzKdcQ7fdRAgSMHes9iDBhb0BRn9w5FaB+B2snqwRmbOTnsgqU/ZAp7mZFWzzIj7Z4fSSE/3Iz0XawZvZJVkgGs11gUZUqxvXdVYE8NqW0NFNEvghJWWS/y+tiyFNSD/Sp71FJmKF43O7rW0PtIBx50F3eTe4uG67e+aliJ5XmwduieNrzxZeVKIO1BeSURfUJwRCwF3sL6ZKuXFajgldT1OdC411GNEg3f93Nm7caZEU1r/oWt7bkeS0mSo3fOu6qcdFnkNMoueVt1E2NffL8MyRwuA3Kt4PrVR+ZK7ZCer7rHlMMt43U9vCcqA2t6TKvo4el2fv91TC2PSZU8IiPz1vAIrODhgTSjekdI7Y5ww5hSt8NZteM4q55cr8N7yD+kVkeUSh1T63TEQuQ93j+9QkeocQZX5zi2Nke4VsPBjlRtmFiTI45rCa/GMbkWx/G6tNThOLoKR1xFhtXfmFp9I5Yqg+tuTK+6MVlENjJh9TaOshtbcDhWXOOYk8eBZTXa4XBPk6BBycv0sXyAKkyvg/U9Taz57pGhrcxABI4M4+K/29PkRsG5VWR716ixTfvG2IV4bnTzzMOJL+BqNTcm5/VqMf2ME/rY/Wq9Fc2C6y/nhG6jqf1LRRoZtCddoRcIcWbs6gU5wQBGUJ7FGvyNcZvEIGsgkh2kZTavRKqROWjpLWmDIY83ljYYHDI9ks1Y8VMjMimzKA27q60UYSkhL+SQdMOz9QYR2arOaqO7pGOWdMwYpCUds6RjJiJa0jFLOmZJxyzpmCUdY8Xgrf5X8bfV/vNCmFL3bzAX61fbO26QhH+H809L/05TJBkCmhqNsQ9LgbDnpCUmoPF0wD6ieT3CjsnXEwuWrgoOapqiEOhioflcGDcsRR1RVBP1IKgnSjH4NqS8rW4lXivonAHencVYxkeSAeJ5MZ0NRNCAMcAxM2XqstIPfcav+z79o8tBDcTTXblO7HXrhMSyjHfAuthh4d4taG9AvxG+XchtczQjdFFXfL1Ez5hI/Y8EnhOK/XcoAk7dZ8Dt1XMDUXYINRO7fA8iJjUDde/FIlTCdlDm9wgwFZ/RStiDqqEmmFn6+63SELpoUV3rKpNKadcci90vjBV/w8kj22wu0d8516fBbsosu0Ttv/XnQ9Wqh/FW+8oDXVyzvMhAQnrZSeIaU8rkbUk1C8Yv0T//+etnkmWQ/lQ3f2XtKFPOfIwWltfbj11nHSq6rl3Hk9R+fXOva3+JiqVH701QexZINTtIkZ3hoZx850JGNiwWHBLlCq7Qf67+IwbyFkugQH3Yx+HN3Y7pkvpZ65FVSjz9RVFjIqg3eFcb50frGTQKfHncndqavfuuk7AJZ/QP9hArpKmoRQloBqsv4SENuq5xDGj0lwXnMrDSMQLGuiS4vWeE8OlIoIJlpEepPXSRqKB5xl0qXU6hIqXmRKK7kXpgJEbcKdaiFAXQdHAI3RcaHXA30wmNCRE1Z7XR7SxXF7S2pPk9k5DDuWrBkh0Sg0R/A+EZC2vZ7NZLYSHXjQVEw6GErgvKNzB4Se0dBL6fiL2iPMo+BZxmhLo5j9nczzWBljXeSOBtl9JIEqavYuAqCNxgkhmaCPnH/9I91UuhyNg+n3lRheEYO4JRfGOBLTUfgrvbcPT4bEVacbFNR7oRr8hIgsPngkfhaLggQjdsYiSRgiDcU35p1lzp5w5jt7+m5tihvhAFJHMOzMXC2JUVcejNOPBKzwfL4BUArEit1xhEB1XxGQIyT8BGcg4x62DHTMz4Ux6zJva6FLOZ7UAXkpdwiTY4E6Bm5SV9pOyZuvtNSeuRwmuksxIzGuUBH58zjDnbN47fnm6C3dbDNg/7+mfXTYGnEVAzKqs2mNpSUucrgm3I/KWmcF9cZ6/H5p6tYl4UeY3WXwbMWHY5ie70KfZTmaapGzVHGlfISeHoQ/z9YnR2Ac8ZoWJVuSL224wGb4eHiQrZpxsrsx0Tcn0ajoq0i+3EQXga43qwPK7uzwkXWnow65WW22al5QZoSuh2tVodu8ASE928uKOZn7pj0JhYW242vJdDtP2ZGcSawdYE66IO813BCaeOJlT3HDbGrRgzqh7sDu97reeKBXB0W724s9QKCZ3VvhQufx+Oh0r136nY2IOu93AqodVXvekLlGpO6GGv84kdOL3mz1nWP/WBDrKKD+DzLrGkuCmzbN9wG5Wmsfisjy98K9nBtYnzXItBM4pzOV1u+LbG+j8a61iGuC+lKQgqDoSq2BxSdLHDPNUDlID0J99xkjhLKIcNdS6kyP6dnRNYmC2seo766SX6XTX1d9XW31Vjf3eMH5aGH9E+TU6LsjI/XBQZAYFkl2nskXG9HP5jhoSwKbN4o3FDMdpwbL8eNCyRb7kdV6+i1PeBtjeZogtQvaoy3ru6BX2XfYb44EB4reM7KkQ48aBinNpsxpQDR+0SInqBWGGQ9/YBbMB1o+ap9WyMz4bHeV3qbpVsgH0dam6UGwCsKwMgJOSx/F1V29uoVxjF6Vl2dKIpg4ilNm+7B8O992C5rbL/uAH5qB408s3dVrlcVGkhtxTiNUOkN148c7mT8fBZ7mQMwzNeS/SJZWUea/m+Ivbiex6HUvlXBcwZiCyX5NVPYP9bLsmbKqDlkrzueZeX5N0HXo13hpvo/uG4f64P5Ry39FVBXg3m/wMAAP//yaE+1g==" + return "eJzsXU9v47iSv/enIPqUWWR8WCz2kMMC72Xewza6p1826bw5LBYeRirbnEikhqSS9vv0C1L/aImkKIt20ol0aHQsu+rHqmKxWCSLP6NH2F+hx/IBOAUJ4gNCksgMrtDHz+2HHz8glIJIOCkkYfQK/dcHhBDqvoBykJwk6tccMsACrtAWf0BIgJSEbsUV+t+PQmQfL9HHnZTFx/9T73aMy3XC6IZsr9AGZwI+ILQhkKXiSjP4GVGcQw+eeuS+UBw4K4v6Ews89XyiG8ZzrD5GmKZISCyJkCQRiG1QwVKBckzxFlL0sDf4rGoKJhoTES6IAP4EvH1jA+UB1pPfX24+oYqgIcrmORRp8/ShmfA4/FmCkKskI0DlwVcanI+wf2Y87b3zoFXPtaaH4DskpdJrw0h4UXAQrOQJxMNxW1GGFFlp9wGI8uGUGFzkBzASVsQHgDRZdJFkpZDALzVTUeAELlvp/OTF9QT8IR6s//727QYNSA4sk6URRaF5DkgOeVIJVK4Vo/hqqDFoFmjAoo8l5fs1L2k8GL+B3AFHcgcND1QKECjle9Rn1AfzSGif2wwknwlNlXetqY+oJC8YjeujGpJoh2maKS9lCMWLpu+7ZyJRTl2TRBvWaCbATTwBF4RFNI2aYIti2Mw+BC25g8FtJoSmk9gI95nnIHcsoj3qjmkhOmg0ExHNsG1xn2rDtuAsASGsHG2GaBvvTXpJUa4EJIP3Dc2UlQ9Z3+8NGnJ9c48EJIymfWQdpxxyxvdqWCcpULl62HeR2ZBvxujW8rKKy66Q68cHqP6qvoQIRQ3PGsMYxCfCZYmzcyKsWY4B3KRixQqgq4SVA+83Cu2A9dcyfwCuPK4iiDYkg/YLjLvVKCTmEtIIRnNXGQwShCagXUxt3A0PawdQE4Fo1t+OqyXX0f6qFKsCeAJUkgxW/+ZsIXv4AxKbAqoX6ylyaPp8AwLlJOGs7k6og+PWia0Zosxn6sePKynzMsOSPAGysfJBm2+8DTRNSY9QDf1RIIL8C6qeHVPTU0ArBJPUakD2aTWGQzrAOFHFBsxTaFiR92AQBaMCXlS9FYQp+h2CPr2CTZTBGh4CjaHiGoqd1DDoj29TTcOsI02VBln5+Dt5O4baJvGBsECWLEuvyfGCvIjRgjV3YzLLsASa7I+xZJu2REPwUpmoQlD9TarAyRyTRiHFM6EWE50umIcyeQR51iGnZo12REi25ThHFQg32NBQYgqKhmalyVDlnSZy6LBQMxCuPgwD8wJ67FCHazIpOVd+bL7sPtFNRrY7GWDqjG55SSmh26hTlc5/JnrQUr9GNSN/Vhlkkq4quUfx5F3Sv9amQFhqLlb2uEyJXMGTSxFT2Wt6SNOzt7diyEFBgzQiz4Zkn3k31lCJCZ23xmFIt6UXZYlDzyzXkuT2VG6KZf/FSMLmThFEA4JGeiV4FB/LUN7co1LgLVgE4Wq2CUX/1tkPbYB8VA8aybiN8DjxMQYmE4tT7rNx+JLmGZGv+Vy3Rqekfs041KKnmDoHrAO0mDIlFhfoUcCBYCujgHSEYQuLpbAqrGNSh0okOIN0vckYdn2xmXLUs5wYbVDSxQLhhqb6m210WkgyiTONHeEsYwmW+CED9TtvYzOSE/njtTaFDaGQVvDb7HvnBi/UJ06JILJBJdW/hdS+gJexbXj+eKRVX9hWheEbNtEZ4SdMMmxPQs13SK6ZMArpeWPTaRSuay2dtqkowQVOiNyr0NdOvfWo9TffvnQqSw6XjHJ2b18q2qWHC4UoT+BeqZg3ttujdxRxEPumbaDrJ87mGAshHPwhRyxUilEIIIddxgekTcMC6HANK1rq6H046r4FjizDnS6Ufl0CqcTgbO4rjyt/NdBPDC0d+kevProMafOMALM2CHeMaUqID7YpoDfVR27v7vw9pAH8zPgjoVsB7jTYW5DHb1UzkQAZJpcCb2GDy8ySSJySHrQj6vJWig1y8GlHTfwH42fCo3k5UbW9hzG5ibjP5z3MKG4Zk3oni9gLCfnkycX7CHbsUjLD7/c+B7NLqI68X24udoY5xr1ldmFm9jnLMuDV4YdZGf7rllh9lCJOfv9FtqCec1f6ube5nnl7q/o3HruvOIewXdT/YjQi3090w7GQvExkyWFIfNnMWzVn2cy7bOZdNvMGNGPZzGsHsmzmDca4bOZdNvMum3nnb+a1RJlTt/c+M/74ZwmlPeI8ZuhToEEFnNWWu/nD+ZeKYLu3rh7MfbFESTeEErGLEk7ct8RCWOM0jWHDvzV6UQRHDDmFQu6i8tQUR7uP5CRKf+34mjuYNXX7xIylsErUhD2RzD6/PsZw4YkkOpKIGQPrZYuGss9gd4AzuYuxL7xj3lJF9kTQKfbk+zlVeBxLVeHsbg4WktyNbH0S4BT4ioh1joV05GQeGMsA9wO9sUPru+7UutY1EajH40Mfjd6t+qHPfkLC6tsOzNIb1e7XJmcFahzSfaN9I3dYIswBbYECx7KqFdLsFa796gEHQtXEVgn3c79yCZqQDHMbmEPXXmlfV8Or4oI4JIynopJ7a3yS5FB9VmAuSVJmmFdCQDssEEv0BvTUglD/UuK8sKAcOhNf2m9DuJDrmhV11OuYvrn3WwNQtVPzQB0P9VnfqszDHicHpFiM4OlyIWKwEldhkPBdhlvDrxWd2hIg7YoDkCegFnEkrNivJbMh6MY0LHpTPXfqzYvuVlMKBddaYb/oxpHcv+2LdondzzEHiVMscZDZj+ijooSwECwh2ss8E7nz6sTXkexdcvoI3zohDrif/EG+DhCwUnHQCTQDwqhf8idNMNec/Tx1aZ24jDVJRCh63pFkV7vcZyy6EceKpsmDr6NXDPlnXTHEFIg/7V6SiEsZ95T8WQLSyWGyISpAYAYQS3KgTYNCtllnhD5GBHP7BXEoOAiFpq4m43IIhD6x7AnStQXjqfxCw9MmF5+HwAWJbzl/ufnU1puprcejrriFhxTvx7r40AjjuM6DGs7Dw/R0/bWhPEH0cTvs/adfRnib08850btxoEzPGJazZMtZMscT+yzZV2VvP/YxsmVvue1Z9pb3nnh7y5ctxD3AyxZiO/BlC7FnCzEFqewmmr/m39+08d1CAuRJ52ldtNpsMue29ahAzKF4vrv4tNmat62QbxxTkRMpX49Ovll10qahl/361RMozb8vW/UnCmjZpd89A+G8hw36xkKz4yBwH9Q5TnB3qF7H2e0Oj+v8dhvTlNSZwTnGb5NcRYAnOovvHhPGGYwxQYE9HIWmSEJ6OpqWSvmU64h3+qiBAkcO9J7FGDC2oCnO7h2K0D4CtZPVgzM2c3LYBUt/yBT2MiOtnmVG2j0/kkJ+uBnpu1gzeiWrJANYr7EoypRie++qwJ4aUtsaKKJfBCWssl7k9bFlKagH+1X2qKXMULxudnStofeRDjzoLu4m9xYN12991bASy/Ng7dA9bXjjy8qVQNqD8koi+oTgiFgKvIX1yVYvK1DBK6nrc6Bxr6MaJRq+7+fM2o0zI5rW/Atb23M9lpIkR++cd1U56bLIaZRd8rbqJsa++H4ZkjlcBuRawfWrj8yV2iE9X3WPKYdbxut6eE9UBtb0mFbRw9Pt/P7rmFoekyp5REbmreERWMHDA2lG9Y6Q2h3hhjGlboezasdxVj25Xof3kH9IrY4olTqm1umIhch7vH96hY5Q4wyuznFsbY5wrYaDHanaMLEmRxzXEl6NY3ItjuN1aanDcXQVjriKDKu/MbX6RixVBtfdmF51Y7KIbGTC6m0cZTe24HCsuMYxJ48Dy2q0w+GeJkGDkpfpY/kAVZheB+t7mljz3SNDW5mBCBwZxsV/t6fJjYJzq8j2rlFjm/aDsQvx3OjmmYcTX8DVam5MzuvVYvoZJ/Sx+9V6K5oF11/OCd1GU/vXijQyaE+6Qi8Q4szY1QtyggGMoDyLNfgb4zaJQdZAJDtIy2xeiVQjc9DSW9IGQx5vLG0wOGR6JJux4qdGZFJmURp2V1spwlJCXsgh6YZn6w0islWd1UZ3Sccs6ZgxSEs6ZknHTES0pGOWdMySjlnSMUs6xorBW/2v4m+r/eeFMKXu32Au1q+2d9wgCf8O55+W/o2mSDIENDUaYx+WAmHPSUtMQOPpgH1E83qEHZOvJxYsXRUc1DRFIdDFQvO5MG5YijqiqCbqQVBPlGLwbUh5W91KvFbQOQO8O4uxjI8kA8TzYjobiKABY4BjZsrUZaUf+oxf9336R5eDGoinu3Kd2OvWCYllGe+AdbHDwr1b0N6AfiN8u5Db5mhG6KKu+HqJnjGR+j8SeE4o9t+hCDh1nwG3V88NRNkh1Ezs8j2ImNQM1L0Xi1AJ20GZ3yPAVHxGK2EPqoaaYGbp77dKQ+iiRXWtq0wqpV1zLHZfGCv+ipNHttlcor9xrk+D3ZRZdona/9bvh6pVD+Ot9pUHurhmeZGBhPSyk8Q1ppTJ25JqFoxfon/849fPJMsg/alu/sraUaac+RgtLK+3H7vOOlR0XbuOJ6n9+uZe1/4SFUuP3pug9iyQanaQIjvDQzn5zoWMbFgsOCTKFVyh/1z9RwzkLZZAgfqwj8Obux3TJfWz1iOrlHj6i6LGRFBv8K42zo/WM2gU+PK4O7U1e/ddJ2ETzugf7CFWSFNRixLQDFZfwkMadF3jGNDoLwvOZWClYwSMdUlwe88I4dORQAXLSI9Se+giUUHzjLtUupxCRUrNiUR3I/XASIy4U6xFKQqg6eAQui80OuBuphMaEyJqzmqj21muLmhtSfN7JiGHc9WCJTskBon+BsIzFtay2a2XwkKuGwuIhkMJXReUb2Dwkto7CHw/EXtFeZR9CjjNCHVzHrO5X2oCLWu8kcDbLqWRJExfxcBVELjBJDM0EfIf/5/uqV4KRcb2+cyLKgzH2BGM4hsLbKn5ENzdhqPHZyvSiottOtKNeEVGEhw+FzwKR8MFEbphEyOJFAThnvJLs+ZKv3QYu/01NccO9YUoIJlzYC4Wxq6siENvxoFXej5YBq8AYEVqvcYgOqiKzxCQeQI2knOIWQc7ZmLGn/KYNbHXpZjNbAe6kLyES7TBmQA1Ky/pI2XP1N1vSlqPFF4jnZWY0SgP+PicYczZvnH89nQT7LYetnnY1z+7bgo8jYCaUVm1wdSWkjpfEWxD5i81hfvqOns9NvdsFfOiyGu0/jJgxrLLSXSnT7GfyjRN3ag50rhCTgpHH+LvF6NrBQxcECGByieWlXms4aojiyq6zdiFNpzl+ps/KzcJP3vGNPheACdqqD0QzqkSAv+sgCoSjlStr/+EzWdqHtbagL61jqmNqJYxcJIwnur7bJihHUdcwDjewjrJsOOO+wDudxURpIm0qYGBZZnT4wfIxGA107OWebCSGQDpi+agsVhwBEz87K5oVueIVA+O2O/9GnwcPqFSyD7dWJntmJDr03BUpF1sJ4ar0xjXYeVxFbJOuCTZg1mvSd42a5I3QFWnXq1Wxy5FxkQ3L0JvMjnu2VpMrC03G97LIdp+DgNi5XpqgnX5k/mu4IRJFhOqO9sT4/6YGfVBdoc3I9dZlQI4uq3+uLNU1QnN/7wULn8fjodK9d+p2NiDroxyKqHVlyLqq8ZqTuhhr6OJDpzeHcNZ1j8fhfoBxhl0uymzbN9wG5WmsU1DH/T5s2QHF4zOcy0GzSjO5XSrKLc11v/RWMfWUvpSmoKg4kComsVCii52mKd6gBKQ/uQ7eBVnbnHYUOeSo+zfbjuBhdnCqueon16i31VTf1dt/V019nfH+GFp+BHt0+S0KCvzw0WRERBIsmFg7f/THYjreeOmzOKNxg3FaMOx/SLdsCmi5R5pvd5Y35zb3vmLLkD1qsp47+oW9F32GeKDA+G1ju+oEOHEg4pxvrkZUw4ctUuI6AVihcEKkQ9gA64bNU+tZ2N8NjzO61J3q2QD7OtQc6PcAGBdwQwhIY/l76oq+EZlzyhOz7L3GU0ZRCxVrNvdSu5dOsu9rv3HDchH9aCRb+5e1+VKVwu5pWS1GSK98TKzy+2lh89ye2kYnvGqu1FXDg+XC2cFJHMm7EOpGMuDVm7LdZL1E9j/luskpwpouU6ye97ldZL3gZdInuHOxr87bmrsQznHfZZVkFeD+f8AAAD//y7wGo0=" } diff --git a/metricbeat/module/kubernetes/state_persisentvolume/README.md b/metricbeat/module/kubernetes/state_persistentvolume/README.md similarity index 100% rename from metricbeat/module/kubernetes/state_persisentvolume/README.md rename to metricbeat/module/kubernetes/state_persistentvolume/README.md diff --git a/metricbeat/module/kubernetes/state_persisentvolume/_meta/fields.yml b/metricbeat/module/kubernetes/state_persistentvolume/_meta/fields.yml similarity index 100% rename from metricbeat/module/kubernetes/state_persisentvolume/_meta/fields.yml rename to metricbeat/module/kubernetes/state_persistentvolume/_meta/fields.yml diff --git a/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8 b/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8 new file mode 100644 index 000000000000..08178aeeb107 --- /dev/null +++ b/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8 @@ -0,0 +1,39 @@ +# HELP kube_persistentvolume_status_phase The phase indicates if a volume is available, bound to a claim, or released by a claim. +# TYPE kube_persistentvolume_status_phase gauge +kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Available"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Bound"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Failed"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Pending"} 1 +kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Released"} 0 + +kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Available"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Bound"} 1 +kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Failed"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Pending"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Released"} 0 + +kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Available"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Bound"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Failed"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Pending"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Released"} 1 + +kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Available"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Bound"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Failed"} 1 +kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Pending"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Released"} 0 + +kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Available"} 1 +kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Bound"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Failed"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Pending"} 0 +kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Released"} 0 + +kube_persistentvolume_info{persistentvolume="test-pv-available",storageclass=""} 1 + +kube_persistentvolume_labels{label_app="mysql-server",persistentvolume="test-labeled-pv"} 1 + +kube_persistentvolume_labels{persistentvolume="test-unlabeled-pv"} 1 + +kube_persistentvolume_capacity_bytes{persistentvolume="test-pv"} 5.36870912e+09 diff --git a/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8.expected b/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8.expected new file mode 100644 index 000000000000..ec747fa47ddb --- /dev/null +++ b/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8.expected @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go new file mode 100644 index 000000000000..1c430e9572c0 --- /dev/null +++ b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go @@ -0,0 +1,84 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package state_persistentvolume + +import ( + p "github.com/elastic/beats/metricbeat/helper/prometheus" + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + mb.Registry.MustAddMetricSet("kubernetes", "state_persistentvolume", + NewPersistentVolumeMetricSet, + mb.WithHostParser(p.HostParser)) +} + +// PersistentVolumeMetricSet is a prometheus based MetricSet that looks for +// mb.ModuleDataKey prefixed fields and puts then at the module level +type PersistentVolumeMetricSet struct { + mb.BaseMetricSet + prometheus p.Prometheus + mapping *p.MetricsMapping +} + +// NewPersistentVolumeMetricSet returns a prometheus based metricset for Persistent Volumes +func NewPersistentVolumeMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { + prometheus, err := p.NewPrometheusClient(base) + if err != nil { + return nil, err + } + + return &PersistentVolumeMetricSet{ + BaseMetricSet: base, + prometheus: prometheus, + mapping: &p.MetricsMapping{ + Metrics: map[string]p.MetricMap{ + "kube_persistenvolume_capacity_bytes": p.Metric("capacity.bytes"), + "kube_persistentvolume_status_phase": p.InfoMetric(), + "kube_persistentvolume_labels": p.ExtendedInfoMetric(p.Configuration{StoreNonMappedLabels: true, NonMappedLabelsPlacement: "labels"}), + "kube_persistentvolume_info": p.InfoMetric(), + }, + Labels: map[string]p.LabelMap{ + "persistentvolume": p.KeyLabel("name"), + "phase": p.Label("phase"), + "storageclass": p.Label("storage_class"), + }, + }, + }, nil +} + +// Fetch prometheus metrics and treats those prefixed by mb.ModuleDataKey as +// module rooted fields at the event that gets reported +func (m *PersistentVolumeMetricSet) Fetch(reporter mb.ReporterV2) { + events, err := m.prometheus.GetProcessedMetrics(m.mapping) + if err != nil { + m.Logger().Error(err) + reporter.Error(err) + return + } + + for _, event := range events { + event[mb.NamespaceKey] = "persistentvolume" + reported := reporter.Event(mb.TransformMapStrToEvent("kubernetes", event, nil)) + if !reported { + m.Logger().Debug("error trying to emit event") + return + } + } + return +} diff --git a/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume_test.go b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume_test.go new file mode 100644 index 000000000000..707b5e147962 --- /dev/null +++ b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume_test.go @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package state_persistentvolume + +import ( + "testing" + + "github.com/elastic/beats/metricbeat/helper/prometheus/ptest" +) + +const testFile = "_meta/test/metrics" + +func TestEventMapping(t *testing.T) { + ptest.TestMetricSet(t, "kubernetes", "state_persistentvolume", + ptest.TestCases{ + { + MetricsFile: "./_meta/test/ksm-unit-1.8", + ExpectedFile: "./_meta/test/ksm-unit-1.8.expected", + }, + }, + ) +} From 21b01ebd596ce180c84f1e9e321356f915394003 Mon Sep 17 00:00:00 2001 From: Pablo Mercado Date: Fri, 29 Nov 2019 13:47:56 +0100 Subject: [PATCH 03/10] fix infometric to labelmetric --- .../_meta/test/ksm-unit-1.8 | 14 +-- .../_meta/test/ksm-unit-1.8.expected | 113 +++++++++++++++++- .../state_persistentvolume.go | 12 +- 3 files changed, 124 insertions(+), 15 deletions(-) diff --git a/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8 b/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8 index 08178aeeb107..dc6e8381a867 100644 --- a/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8 +++ b/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8 @@ -5,35 +5,33 @@ kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Bou kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Failed"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Pending"} 1 kube_persistentvolume_status_phase{persistentvolume="test-pv-pending",phase="Released"} 0 - kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Available"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Bound"} 1 kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Failed"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Pending"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-bound",phase="Released"} 0 - kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Available"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Bound"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Failed"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Pending"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-released",phase="Released"} 1 - kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Available"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Bound"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Failed"} 1 kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Pending"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-failed",phase="Released"} 0 - kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Available"} 1 kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Bound"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Failed"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Pending"} 0 kube_persistentvolume_status_phase{persistentvolume="test-pv-available",phase="Released"} 0 - +# HELP kube_persistentvolume_info Information about persistentvolume. +# TYPE kube_persistentvolume_info gauge kube_persistentvolume_info{persistentvolume="test-pv-available",storageclass=""} 1 - +# HELP kube_persistentvolume_labels Kubernetes labels converted to Prometheus labels. +# TYPE kube_persistentvolume_labels gauge kube_persistentvolume_labels{label_app="mysql-server",persistentvolume="test-labeled-pv"} 1 - kube_persistentvolume_labels{persistentvolume="test-unlabeled-pv"} 1 - +# HELP kube_persistentvolume_capacity_bytes Persistentvolume capacity in bytes. +# TYPE kube_persistentvolume_capacity_bytes gauge kube_persistentvolume_capacity_bytes{persistentvolume="test-pv"} 5.36870912e+09 diff --git a/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8.expected b/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8.expected index ec747fa47ddb..6d5bc0ffdf3a 100644 --- a/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8.expected +++ b/metricbeat/module/kubernetes/state_persistentvolume/_meta/test/ksm-unit-1.8.expected @@ -1 +1,112 @@ -null \ No newline at end of file +[ + { + "RootFields": {}, + "ModuleFields": null, + "MetricSetFields": { + "name": "test-pv-bound", + "phase": "Bound" + }, + "Index": "", + "ID": "", + "Namespace": "kubernetes.persistentvolume", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": {}, + "ModuleFields": null, + "MetricSetFields": { + "capacity": { + "bytes": 5368709120 + }, + "name": "test-pv" + }, + "Index": "", + "ID": "", + "Namespace": "kubernetes.persistentvolume", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": {}, + "ModuleFields": null, + "MetricSetFields": { + "name": "test-pv-pending", + "phase": "Pending" + }, + "Index": "", + "ID": "", + "Namespace": "kubernetes.persistentvolume", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": {}, + "ModuleFields": null, + "MetricSetFields": { + "name": "test-pv-available", + "phase": "Available" + }, + "Index": "", + "ID": "", + "Namespace": "kubernetes.persistentvolume", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": {}, + "ModuleFields": null, + "MetricSetFields": { + "name": "test-pv-released", + "phase": "Released" + }, + "Index": "", + "ID": "", + "Namespace": "kubernetes.persistentvolume", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + }, + { + "RootFields": {}, + "ModuleFields": null, + "MetricSetFields": { + "name": "test-pv-failed", + "phase": "Failed" + }, + "Index": "", + "ID": "", + "Namespace": "kubernetes.persistentvolume", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0, + "Period": 0, + "DisableTimeSeries": false + } +] \ No newline at end of file diff --git a/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go index 1c430e9572c0..403668d616c3 100644 --- a/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go +++ b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go @@ -48,15 +48,15 @@ func NewPersistentVolumeMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { prometheus: prometheus, mapping: &p.MetricsMapping{ Metrics: map[string]p.MetricMap{ - "kube_persistenvolume_capacity_bytes": p.Metric("capacity.bytes"), - "kube_persistentvolume_status_phase": p.InfoMetric(), - "kube_persistentvolume_labels": p.ExtendedInfoMetric(p.Configuration{StoreNonMappedLabels: true, NonMappedLabelsPlacement: "labels"}), - "kube_persistentvolume_info": p.InfoMetric(), + "kube_persistentvolume_capacity_bytes": p.Metric("capacity.bytes"), + "kube_persistentvolume_status_phase": p.LabelMetric("phase", "phase"), + "kube_persistentvolume_labels": p.ExtendedInfoMetric(p.Configuration{StoreNonMappedLabels: true, NonMappedLabelsPlacement: "labels"}), + "kube_persistentvolume_info": p.InfoMetric(), }, Labels: map[string]p.LabelMap{ "persistentvolume": p.KeyLabel("name"), - "phase": p.Label("phase"), - "storageclass": p.Label("storage_class"), + + "storageclass": p.Label("storage_class"), }, }, }, nil From 3aef78b0fc85d6e910ceacde6c2c41ed1130cdac Mon Sep 17 00:00:00 2001 From: Pablo Mercado Date: Thu, 5 Dec 2019 10:13:32 +0100 Subject: [PATCH 04/10] format pv label --- .../state_persistentvolume/state_persistentvolume.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go index 403668d616c3..ca4384d7c44e 100644 --- a/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go +++ b/metricbeat/module/kubernetes/state_persistentvolume/state_persistentvolume.go @@ -55,8 +55,7 @@ func NewPersistentVolumeMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { }, Labels: map[string]p.LabelMap{ "persistentvolume": p.KeyLabel("name"), - - "storageclass": p.Label("storage_class"), + "storageclass": p.Label("storage_class"), }, }, }, nil From 2fed72cf67e61434e11d24616e7569fdbfaf2298 Mon Sep 17 00:00:00 2001 From: Pablo Mercado Date: Tue, 10 Dec 2019 18:21:54 +0100 Subject: [PATCH 05/10] add changelog entry --- CHANGELOG.next.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ffcd5e8be95b..f012489a63a3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -540,10 +540,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add module for ActiveMQ. {pull}14580[14580] - Enable script processor. {pull}14711[14711] - Add `kube-state-metrics` `state_service` metrics for kubernetes module. {pull}14794[14794] +- Add `kube-state-metrics` `state_persistentvolume` metrics for kubernetes module. {pull}14859[14859] - Add usage metricset in aws modules. {pull}14925[14925] {issue}14935[14935] - Add billing metricset in aws modules. {pull}14801[14801] {issue}14934[14934] - Add AWS SNS metricset. {pull}14946[14946] + *Packetbeat* - Update DNS protocol plugin to produce events with ECS fields for DNS. {issue}13320[13320] {pull}13354[13354] From a9d85001a6efec4d1a6a9ce1d9f11f81135bb4e5 Mon Sep 17 00:00:00 2001 From: Pablo Mercado Date: Wed, 11 Dec 2019 09:49:59 +0100 Subject: [PATCH 06/10] add conf reference --- metricbeat/docs/modules/kubernetes.asciidoc | 1 + metricbeat/metricbeat.reference.yml | 1 + metricbeat/module/kubernetes/_meta/config.reference.yml | 1 + metricbeat/module/kubernetes/_meta/config.yml | 1 + metricbeat/modules.d/kubernetes.yml.disabled | 1 + 5 files changed, 5 insertions(+) diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 9c5b27279232..1e96d89956ed 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -96,6 +96,7 @@ metricbeat.modules: - state_cronjob - state_resourcequota - state_service + - state_persistentvolume period: 10s hosts: ["kube-state-metrics:8080"] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 9241da7a0661..e14a1ba608e2 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -475,6 +475,7 @@ metricbeat.modules: - state_cronjob - state_resourcequota - state_service + - state_persistentvolume period: 10s hosts: ["kube-state-metrics:8080"] diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index 2f1ff352ceba..3b09dfbbf165 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -36,6 +36,7 @@ - state_cronjob - state_resourcequota - state_service + - state_persistentvolume period: 10s hosts: ["kube-state-metrics:8080"] diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index ddb19d2e1b8d..2df25896c156 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -36,6 +36,7 @@ # - state_cronjob # - state_resourcequota # - state_service +# - state_persistentvolume # period: 10s # hosts: ["kube-state-metrics:8080"] # add_metadata: true diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index bb2ca45e6e8a..a6be340c20dc 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -39,6 +39,7 @@ # - state_cronjob # - state_resourcequota # - state_service +# - state_persistentvolume # period: 10s # hosts: ["kube-state-metrics:8080"] # add_metadata: true From 9138b3bbc5c75024949a5235014b0e2125eb32c7 Mon Sep 17 00:00:00 2001 From: Pablo Mercado Date: Wed, 11 Dec 2019 10:15:00 +0100 Subject: [PATCH 07/10] config reference for x-pack --- x-pack/metricbeat/metricbeat.reference.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index f8a71490f6a4..b4afb155d224 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -616,6 +616,7 @@ metricbeat.modules: - state_cronjob - state_resourcequota - state_service + - state_persistentvolume period: 10s hosts: ["kube-state-metrics:8080"] From eda2f6b7dedccade011265c36d3161e944e8359c Mon Sep 17 00:00:00 2001 From: Pablo Mercado Date: Wed, 11 Dec 2019 19:14:24 +0100 Subject: [PATCH 08/10] format blank lines (probably) better blank line formatting --- metricbeat/helper/prometheus/prometheus.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus.go b/metricbeat/helper/prometheus/prometheus.go index 217f511e2c95..a71a4267198c 100644 --- a/metricbeat/helper/prometheus/prometheus.go +++ b/metricbeat/helper/prometheus/prometheus.go @@ -151,7 +151,6 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS // Convert labels labels := common.MapStr{} keyLabels := common.MapStr{} - for k, v := range allLabels { if l, ok := mapping.Labels[k]; ok { if l.IsKey() { @@ -227,9 +226,8 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS } } } - + return events, nil - } // infoMetricData keeps data about an infoMetric From d295f9534ea15941d358e5b1545e5179c01f55dd Mon Sep 17 00:00:00 2001 From: Pablo Mercado Date: Wed, 11 Dec 2019 19:24:23 +0100 Subject: [PATCH 09/10] some more spacing --- metricbeat/helper/prometheus/prometheus.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus.go b/metricbeat/helper/prometheus/prometheus.go index a71a4267198c..69c959e0b3a9 100644 --- a/metricbeat/helper/prometheus/prometheus.go +++ b/metricbeat/helper/prometheus/prometheus.go @@ -205,7 +205,6 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS event[k] = v } events = append(events, event) - } // fill info from infoMetrics @@ -226,7 +225,7 @@ func (p *prometheus) GetProcessedMetrics(mapping *MetricsMapping) ([]common.MapS } } } - + return events, nil } From b2ddea1a5a1952ad99e90966499226404b3329b9 Mon Sep 17 00:00:00 2001 From: Pablo Mercado Date: Wed, 18 Dec 2019 21:45:16 +0100 Subject: [PATCH 10/10] remove blank line --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4326cd4a0165..6a9bb35edd16 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -561,7 +561,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `index` option to all modules to specify a module-specific output index. {pull}15100[15100] - Add a `system/service` metricset for systemd data. {pull}14206[14206] - *Packetbeat* - Update DNS protocol plugin to produce events with ECS fields for DNS. {issue}13320[13320] {pull}13354[13354]