Adde clustername to all entities. Changed node names to be their ip/hostname (#37)

Author: camdencheek

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.2.0 - 2018-11-29
+### Added
+- Cluster name is now attached to allow entities within a cluster
+### Changed
+- Nodes are no identified by hostname/IP rather than ID
+
 ## 1.1.0 - 2018-11-28
 ### Added
 - ClusterEnvironment argument to help further identify clusters

src/elasticsearch.go

@@ -28,7 +28,7 @@ type argumentList struct {
 
 const (
 	integrationName    = "com.newrelic.elasticsearch"
-	integrationVersion = "1.1.0"
+	integrationVersion = "1.2.0"
 )
 
 var (

src/inventory.go

@@ -53,7 +53,7 @@ func parseLocalNode(nodeStats *LocalNodeResponse) (string, *LocalNode, error) {
 	nodes := nodeStats.Nodes
 	if len(nodes) == 1 {
 		for k := range nodes {
-			return k, nodes[k], nil
+			return *nodes[k].Host, nodes[k], nil
 		}
 	}
 	return "", nil, errors.New("could not identify local node")

src/inventory_definition.go

@@ -7,6 +7,7 @@ type LocalNodeResponse struct {
 
 // LocalNode is the node API object
 type LocalNode struct {
+	Host    *string           `json:"host"`
 	Process *LocalNodeProcess `json:"process"`
 	Ingest  *LocalNodeIngest  `json:"ingest"`
 	Plugins []*LocalNodeAddon `json:"plugins"`

src/inventory_test.go

@@ -155,7 +155,7 @@ func TestGetLocalNode(t *testing.T) {
 	fakeClient.On("Request", "/_nodes/_local").Return(mockedReturnVal, nil).Once()
 
 	resultName, resultStats, _ := getLocalNode(&fakeClient)
-	assert.Equal(t, "z9ZPp87vT92qG1cRVRIcMQ", resultName)
+	assert.Equal(t, "10.77.10.83", resultName)
 
 	actualString, _ := json.Marshal(resultStats)
 	writeGoldenFile(t, goldenPath, actualString)

src/metrics.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"regexp"
 	"strings"
@@ -14,70 +15,72 @@ const indexLimit = 500
 
 // populateMetrics wrapper to call each of the individual populate functions
 func populateMetrics(i *integration.Integration, client Client, env string) {
-	err := populateNodesMetrics(i, client)
+	clusterName, err := populateClusterMetrics(i, client, env)
 	if err != nil {
-		log.Error("There was an error populating metrics for nodes: %v", err)
+		log.Error("There was an error populating metrics for clusters: %v", err)
 	}
 
-	err = populateClusterMetrics(i, client, env)
+	err = populateNodesMetrics(i, client, clusterName)
 	if err != nil {
-		log.Error("There was an error populating metrics for clusters: %v", err)
+		log.Error("There was an error populating metrics for nodes: %v", err)
 	}
 
 	// we want to use the response from common to populate some index-specific stats.
-	commonResponse, err := populateCommonMetrics(i, client)
+	commonResponse, err := populateCommonMetrics(i, client, clusterName)
 	if err != nil {
 		log.Error("There was an error populating metrics for common metrics: %v", err)
 	}
 
 	if args.CollectIndices {
-		err = populateIndicesMetrics(i, client, commonResponse)
+		err = populateIndicesMetrics(i, client, commonResponse, clusterName)
 		if err != nil {
 			log.Error("There was an error populating metrics for indices: %v", err)
 		}
 	}
 }
 
-func populateNodesMetrics(i *integration.Integration, client Client) error {
+func populateNodesMetrics(i *integration.Integration, client Client, clusterName *string) error {
 	log.Info("Collecting node metrics")
 	nodeResponse := new(NodeResponse)
 	err := client.Request(nodeStatsEndpoint, &nodeResponse)
 	if err != nil {
 		return err
 	}
 
-	setNodesMetricsResponse(i, nodeResponse)
+	setNodesMetricsResponse(i, nodeResponse, clusterName)
 	return nil
 }
 
 // setNodesMetricsResponse calls setMetricsResponse for each node in the response
-func setNodesMetricsResponse(integration *integration.Integration, resp *NodeResponse) {
+func setNodesMetricsResponse(integration *integration.Integration, resp *NodeResponse, clusterName *string) {
 	for node := range resp.Nodes {
-		err := setMetricsResponse(integration, resp.Nodes[node], node, "node")
+		err := setMetricsResponse(integration, resp.Nodes[node], *resp.Nodes[node].Host, "node", clusterName)
 		if err != nil {
 			log.Error("There was an error setting metrics for node metrics on %s: %v", node, err)
 		}
 	}
 }
 
-func populateClusterMetrics(i *integration.Integration, client Client, env string) error {
+func populateClusterMetrics(i *integration.Integration, client Client, env string) (*string, error) {
 	log.Info("Collecting cluster metrics.")
 	clusterResponse := new(ClusterResponse)
 	err := client.Request(clusterEndpoint, &clusterResponse)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	if clusterResponse.Name == nil {
-		return fmt.Errorf("cannot set metric response, missing cluster name")
+		return nil, errors.New("cannot set metric response, missing cluster name")
 	}
+
 	if env != "" {
 		*clusterResponse.Name = *clusterResponse.Name + ":" + env
 	}
-	return setMetricsResponse(i, clusterResponse, *clusterResponse.Name, "cluster")
+
+	return clusterResponse.Name, setMetricsResponse(i, clusterResponse, *clusterResponse.Name, "cluster", nil)
 }
 
-func populateCommonMetrics(i *integration.Integration, client Client) (*CommonMetrics, error) {
+func populateCommonMetrics(i *integration.Integration, client Client, clusterName *string) (*CommonMetrics, error) {
 	log.Info("Collecting common metrics.")
 	commonResponse := new(CommonMetrics)
 	err := client.Request(commonStatsEndpoint, &commonResponse)
@@ -86,13 +89,13 @@ func populateCommonMetrics(i *integration.Integration, client Client) (*CommonMe
 	}
 
 	if args.CollectPrimaries {
-		err = setMetricsResponse(i, commonResponse.All, "commonMetrics", "common")
+		err = setMetricsResponse(i, commonResponse.All, "commonMetrics", "common", clusterName)
 	}
 
 	return commonResponse, err
 }
 
-func populateIndicesMetrics(i *integration.Integration, client Client, commonStats *CommonMetrics) error {
+func populateIndicesMetrics(i *integration.Integration, client Client, commonStats *CommonMetrics, clusterName *string) error {
 	log.Info("Collecting indices metrics")
 	indicesStats := make([]*IndexStats, 0)
 	err := client.Request(indicesStatsEndpoint, &indicesStats)
@@ -105,7 +108,7 @@ func populateIndicesMetrics(i *integration.Integration, client Client, commonSta
 		return err
 	}
 
-	setIndicesStatsMetricsResponse(i, indicesStats, commonStats, indexRegex)
+	setIndicesStatsMetricsResponse(i, indicesStats, commonStats, clusterName, indexRegex)
 	return nil
 }
 
@@ -119,7 +122,7 @@ func buildRegex() (indexRegex *regexp.Regexp, err error) {
 	return indexRegex, nil
 }
 
-func setIndicesStatsMetricsResponse(integration *integration.Integration, indexResponse []*IndexStats, commonResponse *CommonMetrics, indexRegex *regexp.Regexp) {
+func setIndicesStatsMetricsResponse(integration *integration.Integration, indexResponse []*IndexStats, commonResponse *CommonMetrics, clusterName *string, indexRegex *regexp.Regexp) {
 	type indexStatsObject struct {
 		name  string
 		stats *IndexStats
@@ -161,7 +164,7 @@ func setIndicesStatsMetricsResponse(integration *integration.Integration, indexR
 	}
 
 	for _, index := range indicesToCollect {
-		if err := setMetricsResponse(integration, index.stats, index.name, "index"); err != nil {
+		if err := setMetricsResponse(integration, index.stats, index.name, "index", clusterName); err != nil {
 			log.Error("There was an error setting metrics for indices metrics: %v", err)
 		}
 	}
@@ -177,16 +180,23 @@ func getIndexFromCommon(indexName string, indexList map[string]*Index) (*Index,
 
 // setMetricsResponse creates an entity and a metric set for the
 // type of response and calls MarshalMetrics using that response
-func setMetricsResponse(integration *integration.Integration, resp interface{}, name string, namespace string) error {
+func setMetricsResponse(integration *integration.Integration, resp interface{}, name string, namespace string, clusterName *string) error {
 	entity, err := integration.Entity(name, namespace)
 	if err != nil {
 		return err
 	}
 
-	metricSet := entity.NewMetricSet(getSampleName(namespace),
-		metric.Attribute{Key: "displayName", Value: entity.Metadata.Name},
-		metric.Attribute{Key: "entityName", Value: entity.Metadata.Namespace + ":" + entity.Metadata.Name},
-	)
+	idAttributes := []metric.Attribute{
+		{Key: "displayName", Value: entity.Metadata.Name},
+		{Key: "entityName", Value: entity.Metadata.Namespace + ":" + entity.Metadata.Name},
+	}
+
+	// If clusterName is non empty apply it
+	if clusterName != nil {
+		idAttributes = append(idAttributes, metric.Attribute{Key: "clusterName", Value: *clusterName})
+	}
+
+	metricSet := entity.NewMetricSet(getSampleName(namespace), idAttributes...)
 
 	return metricSet.MarshalMetrics(resp)
 }

src/metrics_test.go

@@ -12,6 +12,8 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+var testClusterName = "goTestCluster"
+
 type testClient struct {
 	endpointMapping    map[string]string
 	ReturnRequestError bool
@@ -56,7 +58,7 @@ func TestPopulateNodesMetrics(t *testing.T) {
 	client := createNewTestClient()
 	client.init("nodeStatsMetricsResult.json", nodeStatsEndpoint, t)
 
-	populateNodesMetrics(i, client)
+	populateNodesMetrics(i, client, &testClusterName)
 
 	sourceFile := filepath.Join("testdata", "nodeStatsMetricsResult.json")
 	goldenFile, actualContents := createGoldenFile(i, sourceFile)
@@ -76,7 +78,7 @@ func TestPopulateNodesMetrics_Error(t *testing.T) {
 	mockClient.ReturnRequestError = true
 
 	i := getTestingIntegration(t)
-	err := populateNodesMetrics(i, mockClient)
+	err := populateNodesMetrics(i, mockClient, &testClusterName)
 	assert.Error(t, err, "should be an error")
 }
 
@@ -108,7 +110,7 @@ func TestPopulateClusterMetrics_Error(t *testing.T) {
 	mockClient.ReturnRequestError = true
 
 	i := getTestingIntegration(t)
-	err := populateClusterMetrics(i, mockClient, "")
+	_, err := populateClusterMetrics(i, mockClient, "")
 	assert.Error(t, err, "should be an error")
 }
 
@@ -119,7 +121,7 @@ func TestPopulateCommonMetrics(t *testing.T) {
 	args.CollectPrimaries = true
 	client.init("commonMetricsResult.json", commonStatsEndpoint, t)
 
-	populateCommonMetrics(i, client)
+	populateCommonMetrics(i, client, &testClusterName)
 
 	sourceFile := filepath.Join("testdata", "commonMetricsResult.json")
 	goldenFile, actualContents := createGoldenFile(i, sourceFile)
@@ -130,7 +132,7 @@ func TestPopulateCommonMetrics(t *testing.T) {
 	}
 
 	actualLength := len(i.Entities[0].Metrics[0].Metrics)
-	expectedLength := 36
+	expectedLength := 37
 
 	assert.Equal(t, expectedContents, actualContents)
 	assert.Equal(t, expectedLength, actualLength)
@@ -141,7 +143,7 @@ func TestPopulateCommonMetrics_Error(t *testing.T) {
 	mockClient.ReturnRequestError = true
 
 	i := getTestingIntegration(t)
-	_, err := populateCommonMetrics(i, mockClient)
+	_, err := populateCommonMetrics(i, mockClient, &testClusterName)
 	assert.Error(t, err, "should be an error")
 }
 
@@ -154,15 +156,15 @@ func TestPopulateIndicesMetrics(t *testing.T) {
 	commonData, _ := ioutil.ReadFile(filepath.Join("testdata", "indicesMetricsResult_Common.json"))
 	json.Unmarshal(commonData, commonStruct)
 
-	populateIndicesMetrics(i, client, commonStruct)
+	populateIndicesMetrics(i, client, commonStruct, &testClusterName)
 
 	sourceFile := filepath.Join("testdata", "indicesMetricsResult.json")
 	goldenFile, actualContents := createGoldenFile(i, sourceFile)
 
 	for j := range i.Entities {
 		resultStruct := i.Entities[j].Metrics[0].Metrics
 		actualLength := len(resultStruct)
-		expectedLength := 10
+		expectedLength := 11
 		assert.Equal(t, expectedLength, actualLength)
 	}
 
@@ -200,7 +202,7 @@ func TestSetIndicesStatsMetricsResponse_TooManyIndices(t *testing.T) {
 		},
 	}
 
-	setIndicesStatsMetricsResponse(i, indexResponse, commonResponse, nil)
+	setIndicesStatsMetricsResponse(i, indexResponse, commonResponse, &testClusterName, nil)
 
 	// should not collect any entities since there are more than 100 of them.
 	assert.Equal(t, 0, len(i.Entities))
@@ -211,7 +213,7 @@ func TestPopulateIndicesMetrics_Error(t *testing.T) {
 	mockClient.ReturnRequestError = true
 
 	i := getTestingIntegration(t)
-	err := populateIndicesMetrics(i, mockClient, new(CommonMetrics))
+	err := populateIndicesMetrics(i, mockClient, new(CommonMetrics), &testClusterName)
 	assert.Error(t, err, "should be an error")
 }
 
@@ -225,7 +227,7 @@ func TestIndicesRegex(t *testing.T) {
 	commonData, _ := ioutil.ReadFile(filepath.Join("testdata", "indicesMetricsResult_Common.json"))
 	json.Unmarshal(commonData, commonStruct)
 
-	populateIndicesMetrics(i, client, commonStruct)
+	populateIndicesMetrics(i, client, commonStruct, &testClusterName)
 
 	actualLength := len(i.Entities)
 	expectedLength := 1

src/testdata/commonMetricsResult.json.golden

@@ -1 +1 @@
-{"displayName":"commonMetrics","entityName":"common:commonMetrics","event_type":"ElasticsearchCommonSample","primaries.docsDeleted":0,"primaries.docsnumber":2,"primaries.flushTotalTimeInMilliseconds":15,"primaries.flushesTotal":10,"primaries.get.documentsExist":0,"primaries.get.documentsExistInMilliseconds":0,"primaries.get.documentsMissing":0,"primaries.get.documentsMissingInMilliseconds":0,"primaries.get.requests":0,"primaries.get.requestsCurrent":0,"primaries.get.requestsInMilliseconds":0,"primaries.index.docsCurrentlyDeleted":0,"primaries.index.docsCurrentlyDeletedInMilliseconds":0,"primaries.index.docsCurrentlyIndexing":0,"primaries.index.docsCurrentlyIndexingInMilliseconds":67,"primaries.index.docsDeleted":0,"primaries.index.docsTotal":1,"primaries.indexRefreshesTotal":46,"primaries.indexRefreshesTotalInMilliseconds":88,"primaries.merges.current":0,"primaries.merges.docsSegmentsCurrentlyMerged":0,"primaries.merges.docsTotal":0,"primaries.merges.segmentsCurrentlyMergedInBytes":0,"primaries.merges.segmentsTotal":0,"primaries.merges.segmentsTotalInBytes":0,"primaries.merges.segmentsTotalInMilliseconds":0,"primaries.queriesInMilliseconds":0,"primaries.queriesTotal":0,"primaries.queryActive":0,"primaries.queryFetches":0,"primaries.queryFetchesInMilliseconds":0,"primaries.queryFetchesTotal":0,"primaries.sizeInBytes":9364}
+{"clusterName":"goTestCluster","displayName":"commonMetrics","entityName":"common:commonMetrics","event_type":"ElasticsearchCommonSample","primaries.docsDeleted":0,"primaries.docsnumber":2,"primaries.flushTotalTimeInMilliseconds":15,"primaries.flushesTotal":10,"primaries.get.documentsExist":0,"primaries.get.documentsExistInMilliseconds":0,"primaries.get.documentsMissing":0,"primaries.get.documentsMissingInMilliseconds":0,"primaries.get.requests":0,"primaries.get.requestsCurrent":0,"primaries.get.requestsInMilliseconds":0,"primaries.index.docsCurrentlyDeleted":0,"primaries.index.docsCurrentlyDeletedInMilliseconds":0,"primaries.index.docsCurrentlyIndexing":0,"primaries.index.docsCurrentlyIndexingInMilliseconds":67,"primaries.index.docsDeleted":0,"primaries.index.docsTotal":1,"primaries.indexRefreshesTotal":46,"primaries.indexRefreshesTotalInMilliseconds":88,"primaries.merges.current":0,"primaries.merges.docsSegmentsCurrentlyMerged":0,"primaries.merges.docsTotal":0,"primaries.merges.segmentsCurrentlyMergedInBytes":0,"primaries.merges.segmentsTotal":0,"primaries.merges.segmentsTotalInBytes":0,"primaries.merges.segmentsTotalInMilliseconds":0,"primaries.queriesInMilliseconds":0,"primaries.queriesTotal":0,"primaries.queryActive":0,"primaries.queryFetches":0,"primaries.queryFetchesInMilliseconds":0,"primaries.queryFetchesTotal":0,"primaries.sizeInBytes":9364}