Skip to content

Commit 23ad1e7

Browse files
repl-mike-roestmsodiJeff Nadler
authored
Add Gauge for total_fields setting on a index (#411)
* #257: Add gauge for total fields mapped in an index Signed-off-by: Mike Roest <[email protected]> Co-authored-by: Marco Sohns <[email protected]> Co-authored-by: Jeff Nadler <[email protected]>
1 parent 3aab89d commit 23ad1e7

8 files changed

+480
-5
lines changed

‎README.md

+2
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ Further Information
129129
| elasticsearch_indices_indexing_delete_total | counter | 1 | Total indexing deletes
130130
| elasticsearch_indices_indexing_index_time_seconds_total | counter | 1 | Cumulative index time in seconds
131131
| elasticsearch_indices_indexing_index_total | counter | 1 | Total index calls
132+
| elasticsearch_indices_mappings_stats_fields | gauge | | Count of fields currently mapped by index
132133
| elasticsearch_indices_merges_docs_total | counter | 1 | Cumulative docs merged
133134
| elasticsearch_indices_merges_total | counter | 1 | Total merges
134135
| elasticsearch_indices_merges_total_size_bytes_total | counter | 1 | Total merge size in bytes
@@ -151,6 +152,7 @@ Further Information
151152
| elasticsearch_indices_segments_count | gauge | 1 | Count of index segments on this node
152153
| elasticsearch_indices_segments_memory_bytes | gauge | 1 | Current memory size of segments in bytes
153154
| elasticsearch_indices_settings_stats_read_only_indices | gauge | 1 | Count of indices that have read_only_allow_delete=true
155+
| elasticsearch_indices_settings_total_fields | gauge | | Index setting value for index.mapping.total_fields.limit (total allowable mapped fields in a index)
154156
| elasticsearch_indices_shards_docs | gauge | 3 | Count of documents on this shard
155157
| elasticsearch_indices_shards_docs_deleted | gauge | 3 | Count of deleted documents on each shard
156158
| elasticsearch_indices_store_size_bytes | gauge | 1 | Current size of stored index data in bytes

‎collector/indices_mappings.go

+193
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
// Copyright 2021 The Prometheus Authors
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
//
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
package collector
15+
16+
import (
17+
"encoding/json"
18+
"fmt"
19+
"io/ioutil"
20+
"net/http"
21+
"net/url"
22+
"path"
23+
24+
"github.com/go-kit/kit/log"
25+
"github.com/go-kit/kit/log/level"
26+
"github.com/prometheus/client_golang/prometheus"
27+
)
28+
29+
var (
30+
defaultIndicesMappingsLabels = []string{"index"}
31+
)
32+
33+
type indicesMappingsMetric struct {
34+
Type prometheus.ValueType
35+
Desc *prometheus.Desc
36+
Value func(indexMapping IndexMapping) float64
37+
}
38+
39+
// IndicesMappings information struct
40+
type IndicesMappings struct {
41+
logger log.Logger
42+
client *http.Client
43+
url *url.URL
44+
45+
up prometheus.Gauge
46+
totalScrapes, jsonParseFailures prometheus.Counter
47+
48+
metrics []*indicesMappingsMetric
49+
}
50+
51+
// NewIndicesMappings defines Indices IndexMappings Prometheus metrics
52+
func NewIndicesMappings(logger log.Logger, client *http.Client, url *url.URL) *IndicesMappings {
53+
subsystem := "indices_mappings_stats"
54+
55+
return &IndicesMappings{
56+
logger: logger,
57+
client: client,
58+
url: url,
59+
60+
up: prometheus.NewGauge(prometheus.GaugeOpts{
61+
Name: prometheus.BuildFQName(namespace, subsystem, "up"),
62+
Help: "Was the last scrape of the ElasticSearch Indices Mappings endpoint successful.",
63+
}),
64+
totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
65+
Name: prometheus.BuildFQName(namespace, subsystem, "scrapes_total"),
66+
Help: "Current total ElasticSearch Indices Mappings scrapes.",
67+
}),
68+
jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{
69+
Name: prometheus.BuildFQName(namespace, subsystem, "json_parse_failures_total"),
70+
Help: "Number of errors while parsing JSON.",
71+
}),
72+
metrics: []*indicesMappingsMetric{
73+
{
74+
Type: prometheus.GaugeValue,
75+
Desc: prometheus.NewDesc(
76+
prometheus.BuildFQName(namespace, subsystem, "fields"),
77+
"Current number fields within cluster.",
78+
defaultIndicesMappingsLabels, nil,
79+
),
80+
Value: func(indexMapping IndexMapping) float64 {
81+
return countFieldsRecursive(indexMapping.Mappings.Properties, 0)
82+
},
83+
},
84+
},
85+
}
86+
}
87+
88+
func countFieldsRecursive(properties IndexMappingProperties, fieldCounter float64) float64 {
89+
// iterate over all properties
90+
for _, property := range properties {
91+
if property.Type != nil {
92+
// property has a type set - counts as a field
93+
fieldCounter++
94+
95+
// iterate over all fields of that property
96+
for _, field := range property.Fields {
97+
// field has a type set - counts as a field
98+
if field.Type != nil {
99+
fieldCounter++
100+
}
101+
}
102+
}
103+
104+
// count recursively in case the property has more properties
105+
if property.Properties != nil {
106+
fieldCounter = +countFieldsRecursive(property.Properties, fieldCounter)
107+
}
108+
}
109+
110+
return fieldCounter
111+
}
112+
113+
// Describe add Snapshots metrics descriptions
114+
func (im *IndicesMappings) Describe(ch chan<- *prometheus.Desc) {
115+
for _, metric := range im.metrics {
116+
ch <- metric.Desc
117+
}
118+
119+
ch <- im.up.Desc()
120+
ch <- im.totalScrapes.Desc()
121+
ch <- im.jsonParseFailures.Desc()
122+
}
123+
124+
func (im *IndicesMappings) getAndParseURL(u *url.URL) (*IndicesMappingsResponse, error) {
125+
res, err := im.client.Get(u.String())
126+
if err != nil {
127+
return nil, fmt.Errorf("failed to get from %s://%s:%s%s: %s",
128+
u.Scheme, u.Hostname(), u.Port(), u.Path, err)
129+
}
130+
131+
if res.StatusCode != http.StatusOK {
132+
return nil, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode)
133+
}
134+
135+
body, err := ioutil.ReadAll(res.Body)
136+
if err != nil {
137+
_ = level.Warn(im.logger).Log("msg", "failed to read response body", "err", err)
138+
return nil, err
139+
}
140+
141+
err = res.Body.Close()
142+
if err != nil {
143+
_ = level.Warn(im.logger).Log("msg", "failed to close response body", "err", err)
144+
return nil, err
145+
}
146+
147+
var imr IndicesMappingsResponse
148+
if err := json.Unmarshal(body, &imr); err != nil {
149+
im.jsonParseFailures.Inc()
150+
return nil, err
151+
}
152+
153+
return &imr, nil
154+
}
155+
156+
func (im *IndicesMappings) fetchAndDecodeIndicesMappings() (*IndicesMappingsResponse, error) {
157+
u := *im.url
158+
u.Path = path.Join(u.Path, "/_all/_mappings")
159+
return im.getAndParseURL(&u)
160+
}
161+
162+
// Collect gets all indices mappings metric values
163+
func (im *IndicesMappings) Collect(ch chan<- prometheus.Metric) {
164+
165+
im.totalScrapes.Inc()
166+
defer func() {
167+
ch <- im.up
168+
ch <- im.totalScrapes
169+
ch <- im.jsonParseFailures
170+
}()
171+
172+
indicesMappingsResponse, err := im.fetchAndDecodeIndicesMappings()
173+
if err != nil {
174+
im.up.Set(0)
175+
_ = level.Warn(im.logger).Log(
176+
"msg", "failed to fetch and decode cluster mappings stats",
177+
"err", err,
178+
)
179+
return
180+
}
181+
im.up.Set(1)
182+
183+
for _, metric := range im.metrics {
184+
for indexName, mappings := range *indicesMappingsResponse {
185+
ch <- prometheus.MustNewConstMetric(
186+
metric.Desc,
187+
metric.Type,
188+
metric.Value(mappings),
189+
indexName,
190+
)
191+
}
192+
}
193+
}

‎collector/indices_mappings_response.go

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
// Copyright 2021 The Prometheus Authors
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
//
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
package collector
15+
16+
// IndicesMappingsResponse is a representation of elasticsearch mappings for each index
17+
type IndicesMappingsResponse map[string]IndexMapping
18+
19+
// IndexMapping defines the struct of the tree for the mappings of each index
20+
type IndexMapping struct {
21+
Mappings IndexMappings `json:"mappings"`
22+
}
23+
24+
// IndexMappings defines all index mappings
25+
type IndexMappings struct {
26+
Properties IndexMappingProperties `json:"properties"`
27+
}
28+
29+
// IndexMappingProperties defines all the properties of the current mapping
30+
type IndexMappingProperties map[string]*IndexMappingProperty
31+
32+
// IndexMappingFields defines all the fields of the current mapping
33+
type IndexMappingFields map[string]*IndexMappingField
34+
35+
// IndexMappingProperty defines a single property of the current index properties
36+
type IndexMappingProperty struct {
37+
Type *string `json:"type"`
38+
Properties IndexMappingProperties `json:"properties"`
39+
Fields IndexMappingFields `json:"fields"`
40+
}
41+
42+
// IndexMappingField defines a single property of the current index field
43+
type IndexMappingField struct {
44+
Type *string `json:"type"`
45+
Properties IndexMappingProperties `json:"properties"`
46+
Fields IndexMappingFields `json:"fields"`
47+
}

0 commit comments

Comments
 (0)