feat: add spec files for entity modeling

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 .idea
+.vscode
 *.log
 tmp/
 bin/

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/stretchr/objx v0.1.2-0.20180626195558-9e1dfc121bca // indirect
 	github.com/stretchr/testify v1.6.1
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 // indirect
+	gopkg.in/yaml.v2 v2.2.8
 	k8s.io/api v0.16.10
 	k8s.io/apimachinery v0.16.10
 	k8s.io/client-go v0.15.12

go.sum

@@ -218,6 +218,7 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

internal/integration/spec.go

@@ -0,0 +1,114 @@
+// Package integration ...
+// Copyright 2019 New Relic Corporation. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+package integration
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path"
+	"regexp"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+const fileNameMatcher = `^prometheus_.*\.yml$`
+
+// Specs contains all the services specs mapped with the service name
+type Specs struct {
+	SpecsByName map[string]Spec
+}
+
+// Spec contains the rules to group metrics into entities
+type Spec struct {
+	Provider string      `yaml:"provider"`
+	Service  string      `yaml:"service"`
+	Entities []EntityDef `yaml:"entities"`
+}
+
+// EntityDef has info related to each entity
+type EntityDef struct {
+	Type       string        `yaml:"name"`
+	Properties PropertiesDef `yaml:"properties"`
+}
+
+// PropertiesDef defines the dimension used to get entity names
+type PropertiesDef struct {
+	Dimensions []string `yaml:"dimensions"`
+}
+
+// LoadSpecFiles loads all service spec files named like "prometheus_*.yml" that are in the filesPath
+func LoadSpecFiles(filesPath string) (Specs, error) {
+	specs := Specs{SpecsByName: make(map[string]Spec)}
+	var files []string
+
+	filesInPath, err := ioutil.ReadDir(filesPath)
+	if err != nil {
+		return specs, err
+	}
+	for _, f := range filesInPath {
+		if ok, _ := regexp.MatchString(fileNameMatcher, f.Name()); ok {
+			files = append(files, path.Join(filesPath, f.Name()))
+		}
+	}
+
+	for _, file := range files {
+		f, err := ioutil.ReadFile(file)
+		if err != nil {
+			logrus.Errorf("fail to read service spec file %s: %s ", file, err)
+			continue
+		}
+
+		var sd Spec
+		err = yaml.Unmarshal(f, &sd)
+		if err != nil {
+			logrus.Errorf("fail parse service spec file %s: %s", file, err)
+			continue
+		}
+		logrus.Debugf("spec file loaded for service:%s", sd.Service)
+		specs.SpecsByName[sd.Service] = sd
+	}
+
+	return specs, nil
+}
+
+// getEntity returns entity name and type of the metric based on the spec configuration defined for the service.
+// metric example: serviceName_entityName_metricName{dimension="dim"} 0
+// serviceName, entityName and all dimensions defined in the spec should match to get an entity
+func (s *Specs) getEntity(m Metric) (entityName string, entityType string, err error) {
+	res := strings.Split(m.name, "_")
+	// We assume that minimun metric name is composed by "serviceName_entityType"
+	if len(res) < 2 {
+		return "", "", fmt.Errorf("metric: %s has no suffix to identify the entity", m.name)
+	}
+	serviceName := res[0]
+	metricType := res[1]
+
+	var spec Spec
+	var ok bool
+	if spec, ok = s.SpecsByName[serviceName]; !ok {
+		return "", "", fmt.Errorf("no spec files for service: %s", serviceName)
+	}
+	for _, e := range spec.Entities {
+		if metricType == e.Type {
+			entityType = metricType
+			for _, d := range e.Properties.Dimensions {
+				var val interface{}
+				var ok bool
+				// the metric needs all the dimensions defined to avoid entity name collision
+				if val, ok = m.attributes[d]; !ok {
+					return "", "", fmt.Errorf("dimension %s not found in metric %s", d, m.name)
+				}
+				// entity name will be composed by the value of the dimensions defined for the entity in order
+				entityName = entityName + ":" + fmt.Sprintf("%v", val)
+
+			}
+			break
+		}
+	}
+	entityName = strings.TrimPrefix(entityName, ":")
+	return entityName, entityType, nil
+}

internal/integration/spec_test.go

@@ -0,0 +1,110 @@
+// Copyright 2019 New Relic Corporation. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+package integration
+
+import (
+	"testing"
+
+	"github.com/newrelic/nri-prometheus/internal/pkg/labels"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadSpecFilesOk(t *testing.T) {
+	specs, err := LoadSpecFiles("./test/")
+	assert.NoError(t, err)
+	assert.Contains(t, specs.SpecsByName, "ibmmq")
+	assert.Contains(t, specs.SpecsByName, "ravendb")
+}
+func TestLoadSpecFilesNoFiles(t *testing.T) {
+	specs, err := LoadSpecFiles(".")
+	assert.NoError(t, err)
+	assert.Len(t, specs.SpecsByName, 0)
+}
+
+func TestSpecs_getEntity(t *testing.T) {
+
+	specs, err := LoadSpecFiles("./test/")
+	assert.NoError(t, err)
+	assert.Contains(t, specs.SpecsByName, "ravendb")
+
+	type fields struct {
+		SpecsByName map[string]Spec
+	}
+	type args struct {
+		m Metric
+	}
+	tests := []struct {
+		name           string
+		fields         fields
+		args           args
+		wantEntityName string
+		wantEntityType string
+		wantErr        bool
+	}{
+		{
+			name:   "matchEntity",
+			fields: fields{specs.SpecsByName},
+			args: args{
+				Metric{
+					name: "ravendb_database_document_put_bytes_total",
+					attributes: labels.Set{
+						"database": "test",
+					},
+				}},
+			wantEntityName: "test",
+			wantEntityType: "database",
+			wantErr:        false,
+		},
+		{
+			name:   "matchEntityConcatenatedName",
+			fields: fields{specs.SpecsByName},
+			args: args{
+				Metric{
+					name: "ravendb_testentity_document_put_bytes_total",
+					attributes: labels.Set{
+						"dim1": "first",
+						"dim2": "second",
+					},
+				}},
+			wantEntityName: "first:second",
+			wantEntityType: "testentity",
+			wantErr:        false,
+		},
+		{
+			name:    "missingDimentions",
+			fields:  fields{specs.SpecsByName},
+			args:    args{Metric{name: "ravendb_database_document_put_bytes_total"}},
+			wantErr: true,
+		},
+		{
+			name:    "serviceNotDefined",
+			fields:  fields{specs.SpecsByName},
+			args:    args{Metric{name: "service_metric_undefined"}},
+			wantErr: true,
+		},
+		{
+			name:    "shortMetricName",
+			fields:  fields{specs.SpecsByName},
+			args:    args{Metric{name: "shortname"}},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := &Specs{
+				SpecsByName: tt.fields.SpecsByName,
+			}
+			gotEntityName, gotEntityType, err := s.getEntity(tt.args.m)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Specs.getEntity() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if gotEntityName != tt.wantEntityName {
+				t.Errorf("Specs.getEntity() gotEntityName = %v, want %v", gotEntityName, tt.wantEntityName)
+			}
+			if gotEntityType != tt.wantEntityType {
+				t.Errorf("Specs.getEntity() gotEntityType = %v, want %v", gotEntityType, tt.wantEntityType)
+			}
+		})
+	}
+}

internal/integration/test/prometheus_ibmmq.yml

@@ -0,0 +1,18 @@
+provider: prometheus
+service: ibmmq
+display_name: IBM MQ
+entities:
+  - name: qmgr
+    display_name: Queue Manager
+    properties:
+      dimensions: [ qmgr, platform ]
+    metrics:
+      - provider_name: ibmmq_qmgr_alter_durable_subscription_count
+        description: Alter durable subscription count
+        unit: Count
+      - provider_name: ibmmq_qmgr_channel_initiator_status
+        description: Channel Initiator Status
+        unit: Gauge
+      - provider_name: ibmmq_qmgr_commit_count
+        description: Commit count
+        unit: Count

internal/integration/test/prometheus_ravendb.yml

@@ -0,0 +1,27 @@
+provider: prometheus
+service: ravendb
+display_name: Raven Db
+entities:
+  - name: database
+    display_name: Database
+    properties:
+      dimensions: [ database ]
+    metrics:
+      - provider_name: ravendb_database_document_put_bytes_total
+        description: Database document put bytes
+        unit: Count
+      - provider_name: ravendb_database_document_put_total
+        description: Database document puts count
+        unit: Count
+  - name: testentity
+    display_name: testEntity
+    properties:
+      dimensions: [ dim1, dim2 ]
+    metrics:
+      - provider_name: ravendb_testentity_document_put_bytes_total
+        description: Database document put bytes
+        unit: Count
+      - provider_name: ravendb_testentity_document_put_total
+        description: Database document puts count
+        unit: Count
+