From 489091cc8e9b1f4a1435ba99659069545f2c160d Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Thu, 14 Nov 2024 17:26:22 +0300 Subject: [PATCH] PMM-9870 fix collstats indexSizes metrics. (#953) * PMM-9870 fix collstats indexSizes metrics. * PMM-9870 fix collstats indexSizes metrics. * PMM-9870 fix collstats indexDetails metrics. * PMM-9870 fix linter. * PMM-9870 improve help. * PMM-9870 fix test and linters. --- exporter/collstats_collector.go | 13 +-- exporter/collstats_collector_test.go | 15 ++- exporter/common.go | 26 ++--- exporter/dbstats_collector_test.go | 9 +- exporter/diagnostic_data_collector_test.go | 5 +- exporter/indexstats_collector_test.go | 4 +- exporter/metrics.go | 109 ++++++++++++++++----- exporter/profile_status_collector_test.go | 2 +- exporter/replset_config_collector_test.go | 2 +- main.go | 12 ++- 10 files changed, 134 insertions(+), 63 deletions(-) diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index 65030610f..882260765 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -99,23 +99,16 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { aggregation := bson.D{ { - Key: "$collStats", Value: bson.M{ + Key: "$collStats", + Value: bson.M{ // TODO: PMM-9568 : Add support to handle histogram metrics "latencyStats": bson.M{"histograms": false}, "storageStats": bson.M{"scale": 1}, }, }, } - project := bson.D{ - { - Key: "$project", Value: bson.M{ - "storageStats.wiredTiger": 0, - "storageStats.indexDetails": 0, - }, - }, - } - cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, mongo.Pipeline{aggregation, project}) + cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, mongo.Pipeline{aggregation}) if err != nil { logger.Errorf("cannot get $collstats cursor for collection %s.%s: %s", database, collection, err) diff --git a/exporter/collstats_collector_test.go b/exporter/collstats_collector_test.go index 20151ae3c..8da41c9e8 100644 --- a/exporter/collstats_collector_test.go +++ b/exporter/collstats_collector_test.go @@ -53,21 +53,27 @@ func TestCollStatsCollector(t *testing.T) { ti := labelsGetterMock{} collection := []string{"testdb.testcol_00", "testdb.testcol_01", "testdb.testcol_02"} - c := newCollectionStatsCollector(ctx, client, logrus.New(), false, false, ti, collection) + logger := logrus.New() + c := newCollectionStatsCollector(ctx, client, logger, false, false, ti, collection) // The last \n at the end of this string is important expected := strings.NewReader(` -# HELP mongodb_collstats_latencyStats_commands_latency collstats.latencyStats.commands. +# HELP mongodb_collstats_latencyStats_commands_latency collstats.latencyStats.commands.latency # TYPE mongodb_collstats_latencyStats_commands_latency untyped mongodb_collstats_latencyStats_commands_latency{collection="testcol_00",database="testdb"} 0 mongodb_collstats_latencyStats_commands_latency{collection="testcol_01",database="testdb"} 0 mongodb_collstats_latencyStats_commands_latency{collection="testcol_02",database="testdb"} 0 -# HELP mongodb_collstats_latencyStats_transactions_ops collstats.latencyStats.transactions. +# HELP mongodb_collstats_latencyStats_transactions_ops collstats.latencyStats.transactions.ops # TYPE mongodb_collstats_latencyStats_transactions_ops untyped mongodb_collstats_latencyStats_transactions_ops{collection="testcol_00",database="testdb"} 0 mongodb_collstats_latencyStats_transactions_ops{collection="testcol_01",database="testdb"} 0 mongodb_collstats_latencyStats_transactions_ops{collection="testcol_02",database="testdb"} 0 -# HELP mongodb_collstats_storageStats_capped collstats.storageStats. +# HELP mongodb_collstats_storageStats_indexSizes collstats.storageStats.indexSizes +# TYPE mongodb_collstats_storageStats_indexSizes untyped +mongodb_collstats_storageStats_indexSizes{collection="testcol_00",database="testdb",index_name="_id_"} 4096 +mongodb_collstats_storageStats_indexSizes{collection="testcol_01",database="testdb",index_name="_id_"} 4096 +mongodb_collstats_storageStats_indexSizes{collection="testcol_02",database="testdb",index_name="_id_"} 4096 +# HELP mongodb_collstats_storageStats_capped collstats.storageStats.capped # TYPE mongodb_collstats_storageStats_capped untyped mongodb_collstats_storageStats_capped{collection="testcol_00",database="testdb"} 0 mongodb_collstats_storageStats_capped{collection="testcol_01",database="testdb"} 0 @@ -81,6 +87,7 @@ mongodb_collstats_storageStats_capped{collection="testcol_02",database="testdb"} filter := []string{ "mongodb_collstats_latencyStats_commands_latency", "mongodb_collstats_storageStats_capped", + "mongodb_collstats_storageStats_indexSizes", "mongodb_collstats_latencyStats_transactions_ops", } err := testutil.CollectAndCompare(c, expected, filter...) diff --git a/exporter/common.go b/exporter/common.go index 026453a11..6374e2ff5 100644 --- a/exporter/common.go +++ b/exporter/common.go @@ -79,7 +79,10 @@ func listCollections(ctx context.Context, client *mongo.Client, database string, // // - exclude: List of databases to be excluded. Useful to ignore system databases. func databases(ctx context.Context, client *mongo.Client, filterInNamespaces []string, exclude []string) ([]string, error) { - opts := &options.ListDatabasesOptions{NameOnly: pointer.ToBool(true), AuthorizedDatabases: pointer.ToBool(true)} + opts := &options.ListDatabasesOptions{ + NameOnly: pointer.ToBool(true), + AuthorizedDatabases: pointer.ToBool(true), + } filter := bson.D{} @@ -100,24 +103,27 @@ func databases(ctx context.Context, client *mongo.Client, filterInNamespaces []s } func makeExcludeFilter(exclude []string) *primitive.E { - filterExpressions := []bson.D{} + if len(exclude) == 0 { + return nil + } + + filterExpressions := make([]bson.D, 0, len(exclude)) for _, dbname := range exclude { filterExpressions = append(filterExpressions, bson.D{{Key: "name", Value: bson.D{{Key: "$ne", Value: dbname}}}}, ) } - if len(filterExpressions) == 0 { - return nil - } - return &primitive.E{Key: "$and", Value: filterExpressions} } func makeDBsFilter(filterInNamespaces []string) *primitive.E { - filterExpressions := []bson.D{} - nss := removeEmptyStrings(filterInNamespaces) + if len(nss) == 0 { + return nil + } + + filterExpressions := make([]bson.D, 0, len(nss)) for _, namespace := range nss { parts := strings.Split(namespace, ".") filterExpressions = append(filterExpressions, @@ -125,10 +131,6 @@ func makeDBsFilter(filterInNamespaces []string) *primitive.E { ) } - if len(filterExpressions) == 0 { - return nil - } - return &primitive.E{Key: "$or", Value: filterExpressions} } diff --git a/exporter/dbstats_collector_test.go b/exporter/dbstats_collector_test.go index e039f11cb..8b01d8bcf 100644 --- a/exporter/dbstats_collector_test.go +++ b/exporter/dbstats_collector_test.go @@ -58,15 +58,16 @@ func TestDBStatsCollector(t *testing.T) { ti := labelsGetterMock{} - c := newDBStatsCollector(ctx, client, logrus.New(), false, ti, []string{dbName}, false) + logger := logrus.New() + c := newDBStatsCollector(ctx, client, logger, false, ti, []string{dbName}, false) expected := strings.NewReader(` - # HELP mongodb_dbstats_collections dbstats. + # HELP mongodb_dbstats_collections dbstats.collections # TYPE mongodb_dbstats_collections untyped mongodb_dbstats_collections{database="testdb"} 3 - # HELP mongodb_dbstats_indexes dbstats. + # HELP mongodb_dbstats_indexes dbstats.indexes # TYPE mongodb_dbstats_indexes untyped mongodb_dbstats_indexes{database="testdb"} 3 - # HELP mongodb_dbstats_objects dbstats. + # HELP mongodb_dbstats_objects dbstats.objects # TYPE mongodb_dbstats_objects untyped mongodb_dbstats_objects{database="testdb"} 30` + "\n") diff --git a/exporter/diagnostic_data_collector_test.go b/exporter/diagnostic_data_collector_test.go index 2f6f8a334..c3a618681 100644 --- a/exporter/diagnostic_data_collector_test.go +++ b/exporter/diagnostic_data_collector_test.go @@ -58,10 +58,10 @@ func TestDiagnosticDataCollector(t *testing.T) { // The last \n at the end of this string is important expectedString := fmt.Sprintf(` - # HELP mongodb_oplog_stats_wt_btree_fixed_record_size %s.btree. + # HELP mongodb_oplog_stats_wt_btree_fixed_record_size %s.btree.fixed-record size # TYPE mongodb_oplog_stats_wt_btree_fixed_record_size untyped mongodb_oplog_stats_wt_btree_fixed_record_size 0 - # HELP mongodb_oplog_stats_wt_transaction_update_conflicts %s.transaction. + # HELP mongodb_oplog_stats_wt_transaction_update_conflicts %s.transaction.update conflicts # TYPE mongodb_oplog_stats_wt_transaction_update_conflicts untyped mongodb_oplog_stats_wt_transaction_update_conflicts 0`, prefix, prefix) expected := strings.NewReader(expectedString + "\n") @@ -211,6 +211,7 @@ func TestAllDiagnosticDataCollectorMetrics(t *testing.T) { client := tu.DefaultTestClient(ctx, t) logger := logrus.New() + logger.SetLevel(logrus.DebugLevel) ti := newTopologyInfo(ctx, client, logger) dbBuildInfo, err := retrieveMongoDBBuildInfo(ctx, client, logger.WithField("component", "test")) diff --git a/exporter/indexstats_collector_test.go b/exporter/indexstats_collector_test.go index 48d930c0f..1669649d7 100644 --- a/exporter/indexstats_collector_test.go +++ b/exporter/indexstats_collector_test.go @@ -68,7 +68,7 @@ func TestIndexStatsCollector(t *testing.T) { // The last \n at the end of this string is important expected := strings.NewReader(` -# HELP mongodb_indexstats_accesses_ops indexstats.accesses. +# HELP mongodb_indexstats_accesses_ops indexstats.accesses.ops # TYPE mongodb_indexstats_accesses_ops untyped mongodb_indexstats_accesses_ops{collection="testcol_00",database="testdb",key_name="_id_"} 0 mongodb_indexstats_accesses_ops{collection="testcol_00",database="testdb",key_name="idx_01"} 0 @@ -118,7 +118,7 @@ func TestDescendingIndexOverride(t *testing.T) { // The last \n at the end of this string is important expected := strings.NewReader(` - # HELP mongodb_indexstats_accesses_ops indexstats.accesses. + # HELP mongodb_indexstats_accesses_ops indexstats.accesses.ops # TYPE mongodb_indexstats_accesses_ops untyped mongodb_indexstats_accesses_ops{collection="testcol_00",database="testdb",key_name="_id_"} 0 mongodb_indexstats_accesses_ops{collection="testcol_00",database="testdb",key_name="f1_1"} 0 diff --git a/exporter/metrics.go b/exporter/metrics.go index 33c25820b..74e1ea6d1 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -93,24 +93,68 @@ var ( // mongodb_ss_opcounters{legacy_op_type="command"} 67923 // nodeToPDMetrics = map[string]string{ - "collStats.storageStats.indexDetails.": "index_name", - "globalLock.activeQueue.": "count_type", - "globalLock.locks.": "lock_type", - "serverStatus.asserts.": "assert_type", - "serverStatus.connections.": "conn_type", - "serverStatus.globalLock.currentQueue.": "count_type", - "serverStatus.metrics.commands.": "cmd_name", - "serverStatus.metrics.cursor.open.": "csr_type", - "serverStatus.metrics.document.": "doc_op_type", - "serverStatus.opLatencies.": "op_type", - "serverStatus.opReadConcernCounters.": "concern_type", - "serverStatus.opcounters.": "legacy_op_type", - "serverStatus.opcountersRepl.": "legacy_op_type", - "serverStatus.transactions.commitTypes.": "commit_type", - "serverStatus.wiredTiger.concurrentTransactions.": "txn_rw_type", - "serverStatus.queues.execution.": "txn_rw_type", - "serverStatus.wiredTiger.perf.": "perf_bucket", - "systemMetrics.disks.": "device_name", + "collStats.storageStats.indexDetails.": "index_name", + "globalLock.activeQueue.": "count_type", + "globalLock.locks.": "lock_type", + "serverStatus.asserts.": "assert_type", + "serverStatus.connections.": "conn_type", + "serverStatus.globalLock.currentQueue.": "count_type", + "serverStatus.metrics.commands.": "cmd_name", + "serverStatus.metrics.cursor.open.": "csr_type", + "serverStatus.metrics.document.": "doc_op_type", + "serverStatus.opLatencies.": "op_type", + "serverStatus.opReadConcernCounters.": "concern_type", + "serverStatus.opcounters.": "legacy_op_type", + "serverStatus.opcountersRepl.": "legacy_op_type", + "serverStatus.transactions.commitTypes.": "commit_type", + "serverStatus.wiredTiger.concurrentTransactions.": "txn_rw_type", + "serverStatus.queues.execution.": "txn_rw_type", + "serverStatus.wiredTiger.perf.": "perf_bucket", + "systemMetrics.disks.": "device_name", + "collstats.storageStats.indexSizes.": "index_name", + "config.transactions.stats.storageStats.indexSizes.": "index_name", + "config.image_collection.stats.storageStats.indexSizes.": "index_name", + } + + // This map is used to add labels to some specific metrics. + // The difference from the case above that it works with middle nodes in the structure. + // For example, the fields under the storageStats.indexDetails. structure have this + // signature: + // + // "storageStats": primitive.M{ + // "indexDetails": primitive.M{ + // "_id_": primitive.M{ + // "LSM": primitive.M{ + // "bloom filter false positives": int32(0), + // "bloom filter hits": int32(0), + // "bloom filter misses": int32(0), + // ... + // }, + // "block-manager": primitive.M{ + // "allocations requiring file extension": int32(0), + // ... + // }, + // ... + // }, + // "name_1": primitive.M{ + // ... + // }, + // ... + // }, + // }, + // + // Applying the renaming rules, storageStats will become storageStats but instead of having metrics + // with the form storageStats.indexDetails.. where index_name is each one of + // the fields inside the structure (_id_, name_1, etc), those keys will become labels for the same + // metric name. The label name is defined as the value for each metric name in the map and the value + // the label will have is the field name in the structure. Example. + // + // mongodb_storageStats_indexDetails_index_name_LSM_bloom_filter_false_positives{index_name="_id_"} 0 + keyNodesToLabels = map[string]string{ + "storageStats.indexDetails.": "index_name", + "config.image_collection.stats.storageStats.indexDetails.": "index_name", + "config.transactions.stats.storageStats.indexDetails.": "index_name", + "collstats.storageStats.indexDetails.": "index_name", } // Regular expressions used to make the metric name Prometheus-compatible @@ -236,8 +280,11 @@ func rawToPrometheusMetric(rm *rawMetric) (prometheus.Metric, error) { // by prometheus. For first level metrics, there is no prefix so we should use the metric name or // the help would be empty. func metricHelp(prefix, name string) string { + if _, ok := nodeToPDMetrics[prefix]; ok { + return strings.TrimSuffix(prefix, ".") + } if prefix != "" { - return prefix + return prefix + name } return name @@ -251,17 +298,29 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo } for k, val := range m { + nextPrefix := prefix + k + + l := make(map[string]string) + if label, ok := keyNodesToLabels[prefix]; ok { + for k, v := range labels { + l[k] = v + } + l[label] = k + nextPrefix = prefix + label + } else { + l = labels + } switch v := val.(type) { case bson.M: - res = append(res, makeMetrics(prefix+k, v, labels, compatibleMode)...) + res = append(res, makeMetrics(nextPrefix, v, l, compatibleMode)...) case map[string]interface{}: - res = append(res, makeMetrics(prefix+k, v, labels, compatibleMode)...) + res = append(res, makeMetrics(nextPrefix, v, l, compatibleMode)...) case primitive.A: - res = append(res, processSlice(prefix, k, v, labels, compatibleMode)...) + res = append(res, processSlice(nextPrefix, v, l, compatibleMode)...) case []interface{}: continue default: - rm, err := makeRawMetric(prefix, k, v, labels) + rm, err := makeRawMetric(prefix, k, v, l) if err != nil { invalidMetric := prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err) res = append(res, invalidMetric) @@ -302,7 +361,7 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo // Extract maps from arrays. Only some structures like replicasets have arrays of members // and each member is represented by a map[string]interface{}. -func processSlice(prefix, k string, v []interface{}, commonLabels map[string]string, compatibleMode bool) []prometheus.Metric { +func processSlice(prefix string, v []interface{}, commonLabels map[string]string, compatibleMode bool) []prometheus.Metric { metrics := make([]prometheus.Metric, 0) labels := make(map[string]string) for name, value := range commonLabels { @@ -332,7 +391,7 @@ func processSlice(prefix, k string, v []interface{}, commonLabels map[string]str labels["member_idx"] = host } - metrics = append(metrics, makeMetrics(prefix+k, s, labels, compatibleMode)...) + metrics = append(metrics, makeMetrics(prefix, s, labels, compatibleMode)...) } return metrics diff --git a/exporter/profile_status_collector_test.go b/exporter/profile_status_collector_test.go index df47d5a34..d296385f9 100644 --- a/exporter/profile_status_collector_test.go +++ b/exporter/profile_status_collector_test.go @@ -52,7 +52,7 @@ func TestProfileCollector(t *testing.T) { c := newProfileCollector(ctx, client, logrus.New(), false, ti, 30) expected := strings.NewReader(` - # HELP mongodb_profile_slow_query_count profile_slow_query. + # HELP mongodb_profile_slow_query_count profile_slow_query.count # TYPE mongodb_profile_slow_query_count counter mongodb_profile_slow_query_count{database="admin"} 0 mongodb_profile_slow_query_count{database="config"} 0 diff --git a/exporter/replset_config_collector_test.go b/exporter/replset_config_collector_test.go index 5fe456f2a..809f83d35 100644 --- a/exporter/replset_config_collector_test.go +++ b/exporter/replset_config_collector_test.go @@ -40,7 +40,7 @@ func TestReplsetConfigCollector(t *testing.T) { // The last \n at the end of this string is important expected := strings.NewReader(` - # HELP mongodb_rs_cfg_protocolVersion rs_cfg. + # HELP mongodb_rs_cfg_protocolVersion rs_cfg.protocolVersion # TYPE mongodb_rs_cfg_protocolVersion untyped mongodb_rs_cfg_protocolVersion 1` + "\n") // Filter metrics for 2 reasons: diff --git a/main.go b/main.go index a894588d7..c8320e437 100644 --- a/main.go +++ b/main.go @@ -151,11 +151,19 @@ func buildExporter(opts GlobalFlags, uri string, log *logrus.Logger) *exporter.E nodeName = uriParsed.Host } + collStatsNamespaces := []string{} + if opts.CollStatsNamespaces != "" { + collStatsNamespaces = strings.Split(opts.CollStatsNamespaces, ",") + } + indexStatsCollections := []string{} + if opts.IndexStatsCollections != "" { + indexStatsCollections = strings.Split(opts.IndexStatsCollections, ",") + } exporterOpts := &exporter.Opts{ - CollStatsNamespaces: strings.Split(opts.CollStatsNamespaces, ","), + CollStatsNamespaces: collStatsNamespaces, CompatibleMode: opts.CompatibleMode, DiscoveringMode: opts.DiscoveringMode, - IndexStatsCollections: strings.Split(opts.IndexStatsCollections, ","), + IndexStatsCollections: indexStatsCollections, Logger: log, URI: uri, NodeName: nodeName,