Merge pull request #48 from newrelic/unique-keys

Unique keys

Author: camdencheek

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 4.0.0 - 2019-04-22
+### Changed
+- Prefixed namespace to provide better uniqueness
+- Cluster name and environment are now identity attributes
+### Added
+- Added a reporting endpoint to the entities
+
+
 ## 3.0.0 - 2019-02-04
 ### Changed
 - Updated definition version
@@ -65,4 +73,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## 0.1.0 - 2018-08-28
 ### Added
-- Initial version: Includes Metrics and Inventory data
+- Initial version: Includes Metrics and Inventory data

src/elasticsearch.go

@@ -28,7 +28,7 @@ type argumentList struct {
 
 const (
 	integrationName    = "com.newrelic.elasticsearch"
-	integrationVersion = "3.0.0"
+	integrationVersion = "4.0.0"
 )
 
 var (

src/metrics.go

@@ -39,7 +39,7 @@ func populateMetrics(i *integration.Integration, client Client, env string) {
 	}
 }
 
-func populateNodesMetrics(i *integration.Integration, client Client, clusterName *string) error {
+func populateNodesMetrics(i *integration.Integration, client Client, clusterName string) error {
 	log.Info("Collecting node metrics")
 	nodeResponse := new(NodeResponse)
 	err := client.Request(nodeStatsEndpoint, &nodeResponse)
@@ -52,35 +52,31 @@ func populateNodesMetrics(i *integration.Integration, client Client, clusterName
 }
 
 // setNodesMetricsResponse calls setMetricsResponse for each node in the response
-func setNodesMetricsResponse(integration *integration.Integration, resp *NodeResponse, clusterName *string) {
+func setNodesMetricsResponse(integration *integration.Integration, resp *NodeResponse, clusterName string) {
 	for node := range resp.Nodes {
-		err := setMetricsResponse(integration, resp.Nodes[node], *resp.Nodes[node].Name, "node", clusterName)
+		err := setMetricsResponse(integration, resp.Nodes[node], *resp.Nodes[node].Name, "es-node", clusterName)
 		if err != nil {
 			log.Error("There was an error setting metrics for node metrics on %s: %v", node, err)
 		}
 	}
 }
 
-func populateClusterMetrics(i *integration.Integration, client Client, env string) (*string, error) {
+func populateClusterMetrics(i *integration.Integration, client Client, env string) (string, error) {
 	log.Info("Collecting cluster metrics.")
 	clusterResponse := new(ClusterResponse)
 	err := client.Request(clusterEndpoint, &clusterResponse)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
 	if clusterResponse.Name == nil {
-		return nil, errors.New("cannot set metric response, missing cluster name")
-	}
-
-	if env != "" {
-		*clusterResponse.Name = *clusterResponse.Name + ":" + env
+		return "", errors.New("cannot set metric response, missing cluster name")
 	}
 
-	return clusterResponse.Name, setMetricsResponse(i, clusterResponse, *clusterResponse.Name, "cluster", nil)
+	return *clusterResponse.Name, setMetricsResponse(i, clusterResponse, *clusterResponse.Name, "es-cluster", *clusterResponse.Name)
 }
 
-func populateCommonMetrics(i *integration.Integration, client Client, clusterName *string) (*CommonMetrics, error) {
+func populateCommonMetrics(i *integration.Integration, client Client, clusterName string) (*CommonMetrics, error) {
 	log.Info("Collecting common metrics.")
 	commonResponse := new(CommonMetrics)
 	err := client.Request(commonStatsEndpoint, &commonResponse)
@@ -89,13 +85,13 @@ func populateCommonMetrics(i *integration.Integration, client Client, clusterNam
 	}
 
 	if args.CollectPrimaries {
-		err = setMetricsResponse(i, commonResponse.All, "commonMetrics", "common", clusterName)
+		err = setMetricsResponse(i, commonResponse.All, "commonMetrics", "es-common", clusterName)
 	}
 
 	return commonResponse, err
 }
 
-func populateIndicesMetrics(i *integration.Integration, client Client, commonStats *CommonMetrics, clusterName *string) error {
+func populateIndicesMetrics(i *integration.Integration, client Client, commonStats *CommonMetrics, clusterName string) error {
 	log.Info("Collecting indices metrics")
 	indicesStats := make([]*IndexStats, 0)
 	err := client.Request(indicesStatsEndpoint, &indicesStats)
@@ -122,7 +118,7 @@ func buildRegex() (indexRegex *regexp.Regexp, err error) {
 	return indexRegex, nil
 }
 
-func setIndicesStatsMetricsResponse(integration *integration.Integration, indexResponse []*IndexStats, commonResponse *CommonMetrics, clusterName *string, indexRegex *regexp.Regexp) {
+func setIndicesStatsMetricsResponse(integration *integration.Integration, indexResponse []*IndexStats, commonResponse *CommonMetrics, clusterName string, indexRegex *regexp.Regexp) {
 	type indexStatsObject struct {
 		name  string
 		stats *IndexStats
@@ -164,7 +160,7 @@ func setIndicesStatsMetricsResponse(integration *integration.Integration, indexR
 	}
 
 	for _, index := range indicesToCollect {
-		if err := setMetricsResponse(integration, index.stats, index.name, "index", clusterName); err != nil {
+		if err := setMetricsResponse(integration, index.stats, index.name, "es-index", clusterName); err != nil {
 			log.Error("There was an error setting metrics for indices metrics: %v", err)
 		}
 	}
@@ -180,27 +176,29 @@ func getIndexFromCommon(indexName string, indexList map[string]*Index) (*Index,
 
 // setMetricsResponse creates an entity and a metric set for the
 // type of response and calls MarshalMetrics using that response
-func setMetricsResponse(integration *integration.Integration, resp interface{}, name string, namespace string, clusterName *string) error {
-	entity, err := integration.Entity(name, namespace)
+func setMetricsResponse(i *integration.Integration, resp interface{}, name string, namespace string, clusterName string) error {
+	entityIDAttrs := []integration.IDAttribute{
+		{Key: "clusterName", Value: clusterName},
+	}
+	if args.ClusterEnvironment != "" {
+		entityIDAttrs = append(entityIDAttrs, integration.IDAttribute{Key: "env", Value: args.ClusterEnvironment})
+	}
+
+	entity, err := i.EntityReportedVia(fmt.Sprintf("%s:%d", args.Hostname, args.Port), name, namespace, entityIDAttrs...)
 	if err != nil {
 		return err
 	}
 
-	idAttributes := []metric.Attribute{
+	msAttributes := []metric.Attribute{
 		{Key: "displayName", Value: entity.Metadata.Name},
 		{Key: "entityName", Value: entity.Metadata.Namespace + ":" + entity.Metadata.Name},
 	}
 
-	// If clusterName is non empty apply it
-	if clusterName != nil {
-		idAttributes = append(idAttributes, metric.Attribute{Key: "clusterName", Value: *clusterName})
-	}
-
-	metricSet := entity.NewMetricSet(getSampleName(namespace), idAttributes...)
+	metricSet := entity.NewMetricSet(getSampleName(namespace), msAttributes...)
 
 	return metricSet.MarshalMetrics(resp)
 }
 
 func getSampleName(entityType string) string {
-	return fmt.Sprintf("Elasticsearch%sSample", strings.Title(entityType))
+	return fmt.Sprintf("Elasticsearch%sSample", strings.Title(strings.TrimPrefix(entityType, "es-")))
 }

src/metrics_test.go

@@ -58,7 +58,7 @@ func TestPopulateNodesMetrics(t *testing.T) {
 	client := createNewTestClient()
 	client.init("nodeStatsMetricsResult.json", nodeStatsEndpoint, t)
 
-	populateNodesMetrics(i, client, &testClusterName)
+	populateNodesMetrics(i, client, testClusterName)
 
 	sourceFile := filepath.Join("testdata", "nodeStatsMetricsResult.json")
 	goldenFile, actualContents := createGoldenFile(i, sourceFile)
@@ -78,7 +78,7 @@ func TestPopulateNodesMetrics_Error(t *testing.T) {
 	mockClient.ReturnRequestError = true
 
 	i := getTestingIntegration(t)
-	err := populateNodesMetrics(i, mockClient, &testClusterName)
+	err := populateNodesMetrics(i, mockClient, testClusterName)
 	assert.Error(t, err, "should be an error")
 }
 
@@ -99,7 +99,7 @@ func TestPopulateClusterMetrics(t *testing.T) {
 	}
 
 	actualLength := len(i.Entities[0].Metrics[0].Metrics)
-	expectedLength := 11
+	expectedLength := 12
 
 	assert.Equal(t, expectedContents, actualContents)
 	assert.Equal(t, expectedLength, actualLength)
@@ -121,7 +121,7 @@ func TestPopulateCommonMetrics(t *testing.T) {
 	args.CollectPrimaries = true
 	client.init("commonMetricsResult.json", commonStatsEndpoint, t)
 
-	populateCommonMetrics(i, client, &testClusterName)
+	populateCommonMetrics(i, client, testClusterName)
 
 	sourceFile := filepath.Join("testdata", "commonMetricsResult.json")
 	goldenFile, actualContents := createGoldenFile(i, sourceFile)
@@ -143,7 +143,7 @@ func TestPopulateCommonMetrics_Error(t *testing.T) {
 	mockClient.ReturnRequestError = true
 
 	i := getTestingIntegration(t)
-	_, err := populateCommonMetrics(i, mockClient, &testClusterName)
+	_, err := populateCommonMetrics(i, mockClient, testClusterName)
 	assert.Error(t, err, "should be an error")
 }
 
@@ -156,7 +156,7 @@ func TestPopulateIndicesMetrics(t *testing.T) {
 	commonData, _ := ioutil.ReadFile(filepath.Join("testdata", "indicesMetricsResult_Common.json"))
 	json.Unmarshal(commonData, commonStruct)
 
-	populateIndicesMetrics(i, client, commonStruct, &testClusterName)
+	populateIndicesMetrics(i, client, commonStruct, testClusterName)
 
 	sourceFile := filepath.Join("testdata", "indicesMetricsResult.json")
 	goldenFile, actualContents := createGoldenFile(i, sourceFile)
@@ -202,7 +202,7 @@ func TestSetIndicesStatsMetricsResponse_TooManyIndices(t *testing.T) {
 		},
 	}
 
-	setIndicesStatsMetricsResponse(i, indexResponse, commonResponse, &testClusterName, nil)
+	setIndicesStatsMetricsResponse(i, indexResponse, commonResponse, testClusterName, nil)
 
 	// should not collect any entities since there are more than 100 of them.
 	assert.Equal(t, 0, len(i.Entities))
@@ -213,7 +213,7 @@ func TestPopulateIndicesMetrics_Error(t *testing.T) {
 	mockClient.ReturnRequestError = true
 
 	i := getTestingIntegration(t)
-	err := populateIndicesMetrics(i, mockClient, new(CommonMetrics), &testClusterName)
+	err := populateIndicesMetrics(i, mockClient, new(CommonMetrics), testClusterName)
 	assert.Error(t, err, "should be an error")
 }
 
@@ -227,7 +227,7 @@ func TestIndicesRegex(t *testing.T) {
 	commonData, _ := ioutil.ReadFile(filepath.Join("testdata", "indicesMetricsResult_Common.json"))
 	json.Unmarshal(commonData, commonStruct)
 
-	populateIndicesMetrics(i, client, commonStruct, &testClusterName)
+	populateIndicesMetrics(i, client, commonStruct, testClusterName)
 
 	actualLength := len(i.Entities)
 	expectedLength := 1

src/testdata/bad-process-stats.json.golden

@@ -1 +1 @@
-{"name":"Test","protocol_version":"2","integration_version":"0.0.1","data":[{"metrics":[],"inventory":{"config/process/id":{"value":0},"config/process/mlockall":{"value":false}},"events":[]}]}
+{"name":"Test","protocol_version":"3","integration_version":"0.0.1","data":[{"metrics":[],"inventory":{"config/process/id":{"value":0},"config/process/mlockall":{"value":false}},"events":[]}]}

src/testdata/clusterStatsMetricsResult.json.golden

@@ -1 +1 @@
-{"cluster.dataNodes":3,"cluster.nodes":6,"cluster.status":"yellow","displayName":"es6-dev-cluster","entityName":"cluster:es6-dev-cluster","event_type":"ElasticsearchClusterSample","shards.active":30,"shards.initializing":0,"shards.primaryActive":10,"shards.relocating":0,"shards.unassigned":7}
+{"cluster.dataNodes":3,"cluster.nodes":6,"cluster.status":"yellow","displayName":"es6-dev-cluster","entityName":"es-cluster:es6-dev-cluster","event_type":"ElasticsearchClusterSample","reportingEndpoint":":0","shards.active":30,"shards.initializing":0,"shards.primaryActive":10,"shards.relocating":0,"shards.unassigned":7}

src/testdata/commonMetricsResult.json.golden

@@ -1 +1 @@
-{"clusterName":"goTestCluster","displayName":"commonMetrics","entityName":"common:commonMetrics","event_type":"ElasticsearchCommonSample","primaries.docsDeleted":0,"primaries.docsnumber":2,"primaries.flushTotalTimeInMilliseconds":15,"primaries.flushesTotal":10,"primaries.get.documentsExist":0,"primaries.get.documentsExistInMilliseconds":0,"primaries.get.documentsMissing":0,"primaries.get.documentsMissingInMilliseconds":0,"primaries.get.requests":0,"primaries.get.requestsCurrent":0,"primaries.get.requestsInMilliseconds":0,"primaries.index.docsCurrentlyDeleted":0,"primaries.index.docsCurrentlyDeletedInMilliseconds":0,"primaries.index.docsCurrentlyIndexing":0,"primaries.index.docsCurrentlyIndexingInMilliseconds":67,"primaries.index.docsDeleted":0,"primaries.index.docsTotal":1,"primaries.indexRefreshesTotal":46,"primaries.indexRefreshesTotalInMilliseconds":88,"primaries.merges.current":0,"primaries.merges.docsSegmentsCurrentlyMerged":0,"primaries.merges.docsTotal":0,"primaries.merges.segmentsCurrentlyMergedInBytes":0,"primaries.merges.segmentsTotal":0,"primaries.merges.segmentsTotalInBytes":0,"primaries.merges.segmentsTotalInMilliseconds":0,"primaries.queriesInMilliseconds":0,"primaries.queriesTotal":0,"primaries.queryActive":0,"primaries.queryFetches":0,"primaries.queryFetchesInMilliseconds":0,"primaries.queryFetchesTotal":0,"primaries.sizeInBytes":9364}
+{"displayName":"commonMetrics","entityName":"es-common:commonMetrics","event_type":"ElasticsearchCommonSample","primaries.docsDeleted":0,"primaries.docsnumber":2,"primaries.flushTotalTimeInMilliseconds":15,"primaries.flushesTotal":10,"primaries.get.documentsExist":0,"primaries.get.documentsExistInMilliseconds":0,"primaries.get.documentsMissing":0,"primaries.get.documentsMissingInMilliseconds":0,"primaries.get.requests":0,"primaries.get.requestsCurrent":0,"primaries.get.requestsInMilliseconds":0,"primaries.index.docsCurrentlyDeleted":0,"primaries.index.docsCurrentlyDeletedInMilliseconds":0,"primaries.index.docsCurrentlyIndexing":0,"primaries.index.docsCurrentlyIndexingInMilliseconds":67,"primaries.index.docsDeleted":0,"primaries.index.docsTotal":1,"primaries.indexRefreshesTotal":46,"primaries.indexRefreshesTotalInMilliseconds":88,"primaries.merges.current":0,"primaries.merges.docsSegmentsCurrentlyMerged":0,"primaries.merges.docsTotal":0,"primaries.merges.segmentsCurrentlyMergedInBytes":0,"primaries.merges.segmentsTotal":0,"primaries.merges.segmentsTotalInBytes":0,"primaries.merges.segmentsTotalInMilliseconds":0,"primaries.queriesInMilliseconds":0,"primaries.queriesTotal":0,"primaries.queryActive":0,"primaries.queryFetches":0,"primaries.queryFetchesInMilliseconds":0,"primaries.queryFetchesTotal":0,"primaries.sizeInBytes":9364,"reportingEndpoint":":0"}

src/testdata/elasticsearch_sample.yml.golden

@@ -1 +1 @@
-{"name":"Test","protocol_version":"2","integration_version":"0.0.1","data":[{"metrics":[],"inventory":{"config/network.host":{"value":"0.0.0.0"},"config/path.data":{"value":"/var/lib/elasticsearch"},"config/path.logs":{"value":"/var/log/elasticsearch"}},"events":[]}]}
+{"name":"Test","protocol_version":"3","integration_version":"0.0.1","data":[{"metrics":[],"inventory":{"config/network.host":{"value":"0.0.0.0"},"config/path.data":{"value":"/var/lib/elasticsearch"},"config/path.logs":{"value":"/var/log/elasticsearch"}},"events":[]}]}

src/testdata/empty-process-stats.json.golden

@@ -1 +1 @@
-{"name":"Test","protocol_version":"2","integration_version":"0.0.1","data":[{"metrics":[],"inventory":{},"events":[]}]}
+{"name":"Test","protocol_version":"3","integration_version":"0.0.1","data":[{"metrics":[],"inventory":{},"events":[]}]}