From 5563f6f4c321f169cae3942058775acda196b48e Mon Sep 17 00:00:00 2001 From: Guillermo Sanchez Gavier Date: Tue, 25 Aug 2020 17:01:25 +0200 Subject: [PATCH] feat: add spec files for entity modeling --- .gitignore | 1 + go.mod | 1 + go.sum | 1 + internal/integration/spec.go | 114 ++++++++++++++++++ internal/integration/spec_test.go | 110 +++++++++++++++++ .../integration/test/prometheus_ibmmq.yml | 18 +++ .../integration/test/prometheus_ravendb.yml | 27 +++++ 7 files changed, 272 insertions(+) create mode 100644 internal/integration/spec.go create mode 100644 internal/integration/spec_test.go create mode 100644 internal/integration/test/prometheus_ibmmq.yml create mode 100644 internal/integration/test/prometheus_ravendb.yml diff --git a/.gitignore b/.gitignore index 6b285b8b..37194abc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store .idea +.vscode *.log tmp/ bin/ diff --git a/go.mod b/go.mod index 1874d4a9..3588a832 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/stretchr/objx v0.1.2-0.20180626195558-9e1dfc121bca // indirect github.com/stretchr/testify v1.6.1 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 // indirect + gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.16.10 k8s.io/apimachinery v0.16.10 k8s.io/client-go v0.15.12 diff --git a/go.sum b/go.sum index 4ce4c463..2663952e 100644 --- a/go.sum +++ b/go.sum @@ -218,6 +218,7 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/internal/integration/spec.go b/internal/integration/spec.go new file mode 100644 index 00000000..495b5ef1 --- /dev/null +++ b/internal/integration/spec.go @@ -0,0 +1,114 @@ +// Package integration ... +// Copyright 2019 New Relic Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +package integration + +import ( + "fmt" + "io/ioutil" + "path" + "regexp" + "strings" + + "github.com/sirupsen/logrus" + + yaml "gopkg.in/yaml.v2" +) + +const fileNameMatcher = `^prometheus_.*\.yml$` + +// Specs contains all the services specs mapped with the service name +type Specs struct { + SpecsByName map[string]Spec +} + +// Spec contains the rules to group metrics into entities +type Spec struct { + Provider string `yaml:"provider"` + Service string `yaml:"service"` + Entities []EntityDef `yaml:"entities"` +} + +// EntityDef has info related to each entity +type EntityDef struct { + Type string `yaml:"name"` + Properties PropertiesDef `yaml:"properties"` +} + +// PropertiesDef defines the dimension used to get entity names +type PropertiesDef struct { + Dimensions []string `yaml:"dimensions"` +} + +// LoadSpecFiles loads all service spec files named like "prometheus_*.yml" that are in the filesPath +func LoadSpecFiles(filesPath string) (Specs, error) { + specs := Specs{SpecsByName: make(map[string]Spec)} + var files []string + + filesInPath, err := ioutil.ReadDir(filesPath) + if err != nil { + return specs, err + } + for _, f := range filesInPath { + if ok, _ := regexp.MatchString(fileNameMatcher, f.Name()); ok { + files = append(files, path.Join(filesPath, f.Name())) + } + } + + for _, file := range files { + f, err := ioutil.ReadFile(file) + if err != nil { + logrus.Errorf("fail to read service spec file %s: %s ", file, err) + continue + } + + var sd Spec + err = yaml.Unmarshal(f, &sd) + if err != nil { + logrus.Errorf("fail parse service spec file %s: %s", file, err) + continue + } + logrus.Debugf("spec file loaded for service:%s", sd.Service) + specs.SpecsByName[sd.Service] = sd + } + + return specs, nil +} + +// getEntity returns entity name and type of the metric based on the spec configuration defined for the service. +// metric example: serviceName_entityName_metricName{dimension="dim"} 0 +// serviceName, entityName and all dimensions defined in the spec should match to get an entity +func (s *Specs) getEntity(m Metric) (entityName string, entityType string, err error) { + res := strings.Split(m.name, "_") + // We assume that minimun metric name is composed by "serviceName_entityType" + if len(res) < 2 { + return "", "", fmt.Errorf("metric: %s has no suffix to identify the entity", m.name) + } + serviceName := res[0] + metricType := res[1] + + var spec Spec + var ok bool + if spec, ok = s.SpecsByName[serviceName]; !ok { + return "", "", fmt.Errorf("no spec files for service: %s", serviceName) + } + for _, e := range spec.Entities { + if metricType == e.Type { + entityType = metricType + for _, d := range e.Properties.Dimensions { + var val interface{} + var ok bool + // the metric needs all the dimensions defined to avoid entity name collision + if val, ok = m.attributes[d]; !ok { + return "", "", fmt.Errorf("dimension %s not found in metric %s", d, m.name) + } + // entity name will be composed by the value of the dimensions defined for the entity in order + entityName = entityName + ":" + fmt.Sprintf("%v", val) + + } + break + } + } + entityName = strings.TrimPrefix(entityName, ":") + return entityName, entityType, nil +} diff --git a/internal/integration/spec_test.go b/internal/integration/spec_test.go new file mode 100644 index 00000000..e3dcc537 --- /dev/null +++ b/internal/integration/spec_test.go @@ -0,0 +1,110 @@ +// Copyright 2019 New Relic Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +package integration + +import ( + "testing" + + "github.com/newrelic/nri-prometheus/internal/pkg/labels" + "github.com/stretchr/testify/assert" +) + +func TestLoadSpecFilesOk(t *testing.T) { + specs, err := LoadSpecFiles("./test/") + assert.NoError(t, err) + assert.Contains(t, specs.SpecsByName, "ibmmq") + assert.Contains(t, specs.SpecsByName, "ravendb") +} +func TestLoadSpecFilesNoFiles(t *testing.T) { + specs, err := LoadSpecFiles(".") + assert.NoError(t, err) + assert.Len(t, specs.SpecsByName, 0) +} + +func TestSpecs_getEntity(t *testing.T) { + + specs, err := LoadSpecFiles("./test/") + assert.NoError(t, err) + assert.Contains(t, specs.SpecsByName, "ravendb") + + type fields struct { + SpecsByName map[string]Spec + } + type args struct { + m Metric + } + tests := []struct { + name string + fields fields + args args + wantEntityName string + wantEntityType string + wantErr bool + }{ + { + name: "matchEntity", + fields: fields{specs.SpecsByName}, + args: args{ + Metric{ + name: "ravendb_database_document_put_bytes_total", + attributes: labels.Set{ + "database": "test", + }, + }}, + wantEntityName: "test", + wantEntityType: "database", + wantErr: false, + }, + { + name: "matchEntityConcatenatedName", + fields: fields{specs.SpecsByName}, + args: args{ + Metric{ + name: "ravendb_testentity_document_put_bytes_total", + attributes: labels.Set{ + "dim1": "first", + "dim2": "second", + }, + }}, + wantEntityName: "first:second", + wantEntityType: "testentity", + wantErr: false, + }, + { + name: "missingDimentions", + fields: fields{specs.SpecsByName}, + args: args{Metric{name: "ravendb_database_document_put_bytes_total"}}, + wantErr: true, + }, + { + name: "serviceNotDefined", + fields: fields{specs.SpecsByName}, + args: args{Metric{name: "service_metric_undefined"}}, + wantErr: true, + }, + { + name: "shortMetricName", + fields: fields{specs.SpecsByName}, + args: args{Metric{name: "shortname"}}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Specs{ + SpecsByName: tt.fields.SpecsByName, + } + gotEntityName, gotEntityType, err := s.getEntity(tt.args.m) + if (err != nil) != tt.wantErr { + t.Errorf("Specs.getEntity() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotEntityName != tt.wantEntityName { + t.Errorf("Specs.getEntity() gotEntityName = %v, want %v", gotEntityName, tt.wantEntityName) + } + if gotEntityType != tt.wantEntityType { + t.Errorf("Specs.getEntity() gotEntityType = %v, want %v", gotEntityType, tt.wantEntityType) + } + }) + } +} diff --git a/internal/integration/test/prometheus_ibmmq.yml b/internal/integration/test/prometheus_ibmmq.yml new file mode 100644 index 00000000..04084ec9 --- /dev/null +++ b/internal/integration/test/prometheus_ibmmq.yml @@ -0,0 +1,18 @@ +provider: prometheus +service: ibmmq +display_name: IBM MQ +entities: + - name: qmgr + display_name: Queue Manager + properties: + dimensions: [ qmgr, platform ] + metrics: + - provider_name: ibmmq_qmgr_alter_durable_subscription_count + description: Alter durable subscription count + unit: Count + - provider_name: ibmmq_qmgr_channel_initiator_status + description: Channel Initiator Status + unit: Gauge + - provider_name: ibmmq_qmgr_commit_count + description: Commit count + unit: Count diff --git a/internal/integration/test/prometheus_ravendb.yml b/internal/integration/test/prometheus_ravendb.yml new file mode 100644 index 00000000..1c448da7 --- /dev/null +++ b/internal/integration/test/prometheus_ravendb.yml @@ -0,0 +1,27 @@ +provider: prometheus +service: ravendb +display_name: Raven Db +entities: + - name: database + display_name: Database + properties: + dimensions: [ database ] + metrics: + - provider_name: ravendb_database_document_put_bytes_total + description: Database document put bytes + unit: Count + - provider_name: ravendb_database_document_put_total + description: Database document puts count + unit: Count + - name: testentity + display_name: testEntity + properties: + dimensions: [ dim1, dim2 ] + metrics: + - provider_name: ravendb_testentity_document_put_bytes_total + description: Database document put bytes + unit: Count + - provider_name: ravendb_testentity_document_put_total + description: Database document puts count + unit: Count +