feat: add spec files for entity modeling

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 .idea
+.vscode
 *.log
 tmp/
 bin/

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/stretchr/objx v0.1.2-0.20180626195558-9e1dfc121bca // indirect
 	github.com/stretchr/testify v1.6.1
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 // indirect
+	gopkg.in/yaml.v2 v2.2.8
 	k8s.io/api v0.16.10
 	k8s.io/apimachinery v0.16.10
 	k8s.io/client-go v0.15.12

internal/integration/spec.go

@@ -0,0 +1,146 @@
+// Package integration ...
+// Copyright 2019 New Relic Corporation. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+package integration
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"regexp"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+const fileNameMatcher = `^prometheus_.*\.ya?ml$`
+
+// Specs contains all the services specs mapped with the service name
+type Specs struct {
+	SpecsByName map[string]SpecDef
+}
+
+// SpecDef contains the rules to group metrics into entities
+type SpecDef struct {
+	Service  string      `yaml:"service"`
+	Entities []EntityDef `yaml:"entities"`
+}
+
+// EntityDef has info related to each entity
+type EntityDef struct {
+	Type       string        `yaml:"name"`
+	Properties PropertiesDef `yaml:"properties"`
+	Metrics    []MetricDef   `yaml:"metrics"`
+}
+
+// PropertiesDef defines the dimension used to get entity names
+type PropertiesDef struct {
+	Dimensions []string `yaml:"dimensions"`
+}
+
+// MetricDef contains metrics definitions
+type MetricDef struct {
+	Name string `yaml:"provider_name"`
+}
+
+// LoadSpecFiles loads all service spec files named like "prometheus_*.yml" that are in the filesPath
+func LoadSpecFiles(filesPath string) (Specs, error) {
+	specs := Specs{SpecsByName: make(map[string]SpecDef)}
+	var files []string
+
+	filesInPath, err := ioutil.ReadDir(filesPath)
+	if err != nil {
+		return specs, err
+	}
+	for _, f := range filesInPath {
+		if ok, _ := regexp.MatchString(fileNameMatcher, f.Name()); ok {
+			files = append(files, path.Join(filesPath, f.Name()))
+		}
+	}
+
+	for _, file := range files {
+		f, err := ioutil.ReadFile(file)
+		if err != nil {
+			logrus.Errorf("fail to read service spec file %s: %s ", file, err)
+			continue
+		}
+
+		var sd SpecDef
+		err = yaml.Unmarshal(f, &sd)
+		if err != nil {
+			logrus.Errorf("fail parse service spec file %s: %s", file, err)
+			continue
+		}
+		logrus.Debugf("spec file loaded for service:%s", sd.Service)
+		specs.SpecsByName[sd.Service] = sd
+	}
+
+	return specs, nil
+}
+
+// getEntity returns entity name and type of the metric based on the spec configuration defined for the service.
+// conditions for the metric:
+//   - metric.name has to start a prefix that matches with a service from the spec files
+//   - metric.name has to be defined in one of the entities of the spec file
+//   - if dimension has been specified for the entity, the metric need to have all of them.
+//   - metrics that belongs to entities with no dimension specified will share the same name
+func (s *Specs) getEntity(m Metric) (entityName string, entityType string, err error) {
+	spec, err := s.findSpec(m.name)
+	if err != nil {
+		return "", "", err
+	}
+
+	e, ok := spec.findEntity(m.name)
+	if !ok {
+		return "", "", fmt.Errorf("metric: %s is not defined in service:%s", m.name, spec.Service)
+	}
+
+	entityType = strings.Title(spec.Service) + strings.Title(e.Type)
+
+	entityName = e.Type
+
+	for _, d := range e.Properties.Dimensions {
+		var val interface{}
+		var ok bool
+		// the metric needs all the dimensions defined to avoid entity name collision
+		if val, ok = m.attributes[d]; !ok {
+			return "", "", fmt.Errorf("dimension %s not found in metric %s", d, m.name)
+		}
+		// entity name will be composed by the value of the dimensions defined for the entity in order
+		entityName = entityName + ":" + fmt.Sprintf("%v", val)
+	}
+
+	return entityName, entityType, nil
+}
+
+// findSpec parses the metric name to extract the service and resturns the spec definition that matches
+func (s *Specs) findSpec(metricName string) (SpecDef, error) {
+	var spec SpecDef
+
+	res := strings.SplitN(metricName, "_", 2)
+	if len(res) < 2 {
+		return spec, fmt.Errorf("metric: %s has no suffix to identify the entity", metricName)
+	}
+	serviceName := res[0]
+
+	var ok bool
+	if spec, ok = s.SpecsByName[serviceName]; !ok {
+		return spec, fmt.Errorf("no spec files for service: %s", serviceName)
+	}
+
+	return spec, nil
+}
+
+// findEntity returns the entity where metricName is defined
+func (s *SpecDef) findEntity(metricName string) (EntityDef, bool) {
+	for _, e := range s.Entities {
+		for _, em := range e.Metrics {
+			if metricName == em.Name {
+				return e, true
+			}
+		}
+	}
+	return EntityDef{}, false
+}

internal/integration/spec_test.go

@@ -0,0 +1,152 @@
+// Copyright 2019 New Relic Corporation. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+package integration
+
+import (
+	"testing"
+
+	"github.com/newrelic/nri-prometheus/internal/pkg/labels"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadSpecFilesOk(t *testing.T) {
+	specs, err := LoadSpecFiles("./test/")
+	assert.NoError(t, err)
+	assert.Contains(t, specs.SpecsByName, "ibmmq")
+	assert.Contains(t, specs.SpecsByName, "ravendb")
+	assert.Contains(t, specs.SpecsByName, "redis")
+}
+func TestLoadSpecFilesNoFiles(t *testing.T) {
+	specs, err := LoadSpecFiles(".")
+	assert.NoError(t, err)
+	assert.Len(t, specs.SpecsByName, 0)
+}
+
+func TestSpecs_getEntity(t *testing.T) {
+
+	specs, err := LoadSpecFiles("./test/")
+	assert.NoError(t, err)
+
+	type fields struct {
+		SpecsByName map[string]SpecDef
+	}
+	type args struct {
+		m Metric
+	}
+	tests := []struct {
+		name           string
+		fields         fields
+		args           args
+		wantEntityName string
+		wantEntityType string
+		wantErr        bool
+	}{
+		{
+			name:   "matchEntity",
+			fields: fields{specs.SpecsByName},
+			args: args{
+				Metric{
+					name: "ravendb_database_document_put_bytes_total",
+					attributes: labels.Set{
+						"database": "test",
+					},
+				}},
+			wantEntityName: "database:test",
+			wantEntityType: "RavendbDatabase",
+			wantErr:        false,
+		},
+		{
+			name:   "matchEntityWithoutDimensions",
+			fields: fields{specs.SpecsByName},
+			args: args{
+				Metric{
+					name:       "ravendb_document_put_bytes_total",
+					attributes: labels.Set{},
+				}},
+			wantEntityName: "node",
+			wantEntityType: "RavendbNode",
+			wantErr:        false,
+		},
+		{
+			name:   "matchEntityConcatenatedName",
+			fields: fields{specs.SpecsByName},
+			args: args{
+				Metric{
+					name: "ravendb_testentity_document_put_bytes_total",
+					attributes: labels.Set{
+						"dim1": "first",
+						"dim2": "second",
+					},
+				}},
+			wantEntityName: "testentity:first:second",
+			wantEntityType: "RavendbTestentity",
+			wantErr:        false,
+		},
+		{
+			name:   "redisEntityMetric1",
+			fields: fields{specs.SpecsByName},
+			args: args{
+				Metric{
+					name:       "redis_commands_duration_seconds_total",
+					attributes: labels.Set{},
+				}},
+			wantEntityName: "instance",
+			wantEntityType: "RedisInstance",
+			wantErr:        false,
+		},
+		{
+			name:   "redisEntityMetric2",
+			fields: fields{specs.SpecsByName},
+			args: args{
+				Metric{
+					name:       "redis_repl_backlog_is_active",
+					attributes: labels.Set{},
+				}},
+			wantEntityName: "instance",
+			wantEntityType: "RedisInstance",
+			wantErr:        false,
+		},
+		{
+			name:    "missingDimentions",
+			fields:  fields{specs.SpecsByName},
+			args:    args{Metric{name: "ravendb_database_document_put_bytes_total"}},
+			wantErr: true,
+		},
+		{
+			name:    "serviceNotDefined",
+			fields:  fields{specs.SpecsByName},
+			args:    args{Metric{name: "service_metric_undefined"}},
+			wantErr: true,
+		},
+		{
+			name:    "metricNotDefined",
+			fields:  fields{specs.SpecsByName},
+			args:    args{Metric{name: "ravendb_metric_undefined"}},
+			wantErr: true,
+		},
+		{
+			name:    "shortMetricName",
+			fields:  fields{specs.SpecsByName},
+			args:    args{Metric{name: "shortname"}},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := &Specs{
+				SpecsByName: tt.fields.SpecsByName,
+			}
+			gotEntityName, gotEntityType, err := s.getEntity(tt.args.m)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Specs.getEntity() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if gotEntityName != tt.wantEntityName {
+				t.Errorf("Specs.getEntity() gotEntityName = %v, want %v", gotEntityName, tt.wantEntityName)
+			}
+			if gotEntityType != tt.wantEntityType {
+				t.Errorf("Specs.getEntity() gotEntityType = %v, want %v", gotEntityType, tt.wantEntityType)
+			}
+		})
+	}
+}

internal/integration/test/prometheus_ibmmq.yaml

@@ -0,0 +1,20 @@
+service: ibmmq
+display_name: IBM MQ
+entities:
+  - name: qmgr
+    display_name: Queue Manager
+    properties:
+      dimensions: [ qmgr, platform ]
+    metrics:
+      - provider_name: ibmmq_qmgr_alter_durable_subscription_count
+        description: Alter durable subscription count
+        unit: Count
+      - provider_name: ibmmq_qmgr_channel_initiator_status
+        description: Channel Initiator Status
+        unit: Gauge
+      - provider_name: ibmmq_qmgr_commit_count
+        description: Commit count
+        unit: Count
+      - provider_name: go_goroutines
+        description: Number of goroutines that currently exist.
+        unit: gauge

internal/integration/test/prometheus_ravendb.yml

@@ -0,0 +1,35 @@
+service: ravendb
+display_name: Raven Db
+entities:
+  - name: node
+    display_name: RavenDb Node
+    metrics:
+      - provider_name: ravendb_document_put_bytes_total
+        description: Server-wide document put bytes
+        unit: Count
+      - provider_name: ravendb_is_leader
+        description: If 1, then node is the cluster leader, otherwise 0
+        unit: Gauge
+  - name: database
+    display_name: Database
+    properties:
+      dimensions: [ database ]
+    metrics:
+      - provider_name: ravendb_database_document_put_bytes_total
+        description: Database document put bytes
+        unit: Count
+      - provider_name: ravendb_database_document_put_total
+        description: Database document puts count
+        unit: Count
+  - name: testentity
+    display_name: testEntity
+    properties:
+      dimensions: [ dim1, dim2 ]
+    metrics:
+      - provider_name: ravendb_testentity_document_put_bytes_total
+        description: Database document put bytes
+        unit: Count
+      - provider_name: ravendb_testentity_document_put_total
+        description: Database document puts count
+        unit: Count
+

internal/integration/test/prometheus_redis.yml

@@ -0,0 +1,13 @@
+service: redis
+display_name: Redis
+entities:
+  - name: instance
+    display_name: Redis Instance
+    metrics:
+      - provider_name: redis_commands_duration_seconds_total
+        description: Total amount of time in seconds spent per command 
+        unit: Counter
+      - provider_name: redis_repl_backlog_is_active 
+        description: repl_backlog_is_active metric
+        unit: Gauge
+