diff --git a/api/route.go b/api/route.go index 2085741a..756c2f32 100644 --- a/api/route.go +++ b/api/route.go @@ -55,6 +55,20 @@ func (server *Server) GetExecutions(resp http.ResponseWriter, req *http.Request) server.JSONWrite(resp, http.StatusOK, results) } +func (server *Server) GetAccounts(resp http.ResponseWriter, req *http.Request) { + queryLimit, _ := strconv.Atoi(httpparameters.QueryParamWithDefault(req, "querylimit", storage.GetExecutionsQueryLimit)) + params := mux.Vars(req) + executionID := params["executionID"] + accounts, err := server.storage.GetAccounts(executionID, queryLimit) + + if err != nil { + server.JSONWrite(resp, http.StatusInternalServerError, HttpErrorResponse{Error: err.Error()}) + return + + } + server.JSONWrite(resp, http.StatusOK, accounts) +} + // GetResourceData return resuts details by resource type func (server *Server) GetResourceData(resp http.ResponseWriter, req *http.Request) { queryParams := req.URL.Query() diff --git a/api/server.go b/api/server.go index b82d45b1..357c9c69 100644 --- a/api/server.go +++ b/api/server.go @@ -81,6 +81,7 @@ func (server *Server) BindEndpoints() { server.router.HandleFunc("/api/v1/summary/{executionID}", server.GetSummary).Methods("GET") server.router.HandleFunc("/api/v1/executions", server.GetExecutions).Methods("GET") + server.router.HandleFunc("/api/v1/accounts/{executionID}", server.GetAccounts).Methods(("GET")) server.router.HandleFunc("/api/v1/resources/{type}", server.GetResourceData).Methods("GET") server.router.HandleFunc("/api/v1/trends/{type}", server.GetResourceTrends).Methods("GET") server.router.HandleFunc("/api/v1/tags/{executionID}", server.GetExecutionTags).Methods("GET") diff --git a/api/server_test.go b/api/server_test.go index 7ce3ccf0..55192ec4 100644 --- a/api/server_test.go +++ b/api/server_test.go @@ -235,8 +235,57 @@ func TestGetExecutions(t *testing.T) { }) } +} + +func TestGetAccounts(t *testing.T) { + ms, _ := MockServer() + ms.BindEndpoints() + ms.Serve() + testCases := []struct { + endpoint string + expectedStatusCode int + Count int + }{ + {"/api/v1/accounts", http.StatusNotFound, 0}, + {"/api/v1/accounts/1", http.StatusOK, 2}, + {"/api/v1/accounts/err", http.StatusInternalServerError, 0}, + } + + for _, test := range testCases { + t.Run(test.endpoint, func(t *testing.T) { + + rr := httptest.NewRecorder() + req, err := http.NewRequest("GET", test.endpoint, nil) + if err != nil { + t.Fatal(err) + } + ms.Router().ServeHTTP(rr, req) + if rr.Code != test.expectedStatusCode { + t.Fatalf("handler returned wrong status code: got %v want %v", rr.Code, test.expectedStatusCode) + } + + if test.expectedStatusCode == http.StatusOK { + body, err := ioutil.ReadAll(rr.Body) + if err != nil { + t.Fatal(err) + } + + var accountsData []storage.Accounts + + err = json.Unmarshal(body, &accountsData) + if err != nil { + t.Fatalf("Could not parse http response") + } + + if len(accountsData) != test.Count { + t.Fatalf("unexpected accounts data response, got %d expected %d", len(accountsData), test.Count) + } + } + }) + } } + func TestSave(t *testing.T) { ms, mockStorage := MockServer() ms.BindEndpoints() diff --git a/api/storage/elasticsearch/elasticsearch.go b/api/storage/elasticsearch/elasticsearch.go index afda70a2..471d1d55 100644 --- a/api/storage/elasticsearch/elasticsearch.go +++ b/api/storage/elasticsearch/elasticsearch.go @@ -10,6 +10,7 @@ import ( "fmt" "reflect" "strconv" + "strings" "time" elastic "github.com/olivere/elastic/v7" @@ -110,14 +111,24 @@ func (sm *StorageManager) getDynamicMatchQuery(filters map[string]string, operat dynamicMatchQuery := []elastic.Query{} var mq *elastic.MatchQuery for name, value := range filters { - mq = elastic.NewMatchQuery(name, value) - // Minimum number of clauses that must match for a document to be returned - mq.MinimumShouldMatch("100%") - if operator == "and" { - mq = mq.Operator("and") + if name == "Data.AccountID" { + var accountIds = strings.Split(value, ",") + var accountBoolQuery = elastic.NewBoolQuery() + for _, accountId := range accountIds { + accountBoolQuery.Should(elastic.NewMatchQuery(name, accountId)) + } + accountBoolQuery.MinimumShouldMatch("1") + dynamicMatchQuery = append(dynamicMatchQuery, accountBoolQuery) + } else { + mq = elastic.NewMatchQuery(name, value) + // Minimum number of clauses that must match for a document to be returned + mq.MinimumShouldMatch("100%") + if operator == "and" { + mq = mq.Operator("and") + } + log.Info("Query ", mq) + dynamicMatchQuery = append(dynamicMatchQuery, mq) } - - dynamicMatchQuery = append(dynamicMatchQuery, mq) } return dynamicMatchQuery } @@ -183,7 +194,7 @@ func (sm *StorageManager) GetSummary(executionID string, filters map[string]stri for resourceName, resourceData := range summary { filters["ResourceName"] = resourceName log.WithField("filters", filters).Debug("Going to get resources summary details with the following filters") - totalSpent, resourceCount, err := sm.getResourceSummaryDetails(executionID, filters) + totalSpent, resourceCount, spentAccounts, err := sm.getResourceSummaryDetails(executionID, filters) if err != nil { continue @@ -191,8 +202,8 @@ func (sm *StorageManager) GetSummary(executionID string, filters map[string]stri newResourceData := resourceData newResourceData.TotalSpent = totalSpent newResourceData.ResourceCount = resourceCount + newResourceData.SpentAccounts = spentAccounts summary[resourceName] = newResourceData - } return summary, nil @@ -200,43 +211,52 @@ func (sm *StorageManager) GetSummary(executionID string, filters map[string]stri } // getResourceSummaryDetails returns total resource spent and total resources detected -func (sm *StorageManager) getResourceSummaryDetails(executionID string, filters map[string]string) (float64, int64, error) { +func (sm *StorageManager) getResourceSummaryDetails(executionID string, filters map[string]string) (float64, int64, map[string]float64, error) { var totalSpent float64 var resourceCount int64 + var spentAccounts = make(map[string]float64) dynamicMatchQuery := sm.getDynamicMatchQuery(filters, "or") dynamicMatchQuery = append(dynamicMatchQuery, elastic.NewTermQuery("ExecutionID", executionID)) dynamicMatchQuery = append(dynamicMatchQuery, elastic.NewTermQuery("EventType", "resource_detected")) - searchResult, err := sm.client.Search(). + searchResultAccount, err := sm.client.Search(). Query(elastic.NewBoolQuery().Must(dynamicMatchQuery...)). - Aggregation("sum", elastic.NewSumAggregation().Field("Data.PricePerMonth")). + Aggregation("accounts", elastic.NewTermsAggregation().Field("Data.AccountID.keyword"). + SubAggregation("accountSum", elastic.NewSumAggregation().Field("Data.PricePerMonth"))). Size(0).Do(context.Background()) if err != nil { - log.WithError(err).WithFields(log.Fields{ - "filters": filters, - }).Error("error when trying to get summary details") + log.WithError(err).Error("error when trying to get executions collectors") + return totalSpent, resourceCount, spentAccounts, ErrInvalidQuery + } - return totalSpent, resourceCount, err + respAccount, ok := searchResultAccount.Aggregations.Terms("accounts") + if !ok { + log.Error("accounts field term does not exist") + return totalSpent, resourceCount, spentAccounts, ErrAggregationTermNotFound } - log.WithFields(log.Fields{ - "filters": filters, - "milliseconds": searchResult.TookInMillis, - }).Debug("get execution details") + for _, AccountIdBucket := range respAccount.Buckets { - resp, ok := searchResult.Aggregations.Terms("sum") - if ok { - if val, ok := resp.Aggregations["value"]; ok { + spent, ok := AccountIdBucket.Aggregations.Terms("accountSum") + if ok { + if val, ok := spent.Aggregations["value"]; ok { + accountID, ok := AccountIdBucket.Key.(string) + if !ok { + log.Error("type assertion to string failed") + continue + } + spentAccounts[accountID], _ = strconv.ParseFloat(string(val), 64) - totalSpent, _ = strconv.ParseFloat(string(val), 64) - resourceCount = searchResult.Hits.TotalHits.Value + totalSpent += spentAccounts[accountID] + resourceCount += AccountIdBucket.DocCount + } } } - return totalSpent, resourceCount, nil + return totalSpent, resourceCount, spentAccounts, nil } // GetExecutions returns collector executions @@ -298,6 +318,44 @@ func (sm *StorageManager) GetExecutions(queryLimit int) ([]storage.Executions, e return executions, nil } +func (sm *StorageManager) GetAccounts(executionID string, querylimit int) ([]storage.Accounts, error) { + accounts := []storage.Accounts{} + + searchResult, err := sm.client.Search().Query(elastic.NewMatchQuery("ExecutionID", executionID)). + Aggregation("Accounts", elastic.NewTermsAggregation(). + Field("Data.AccountInformation.keyword").Size(querylimit)). + Do(context.Background()) + + if err != nil { + log.WithError(err).Error("error when trying to get AccountIDs") + return accounts, ErrInvalidQuery + } + + resp, ok := searchResult.Aggregations.Terms("Accounts") + if !ok { + log.Error("accounts field term does not exist") + return accounts, ErrAggregationTermNotFound + } + + for _, accountsBucket := range resp.Buckets { + account, ok := accountsBucket.Key.(string) + if !ok { + log.Error("type assertion to string failed") + continue + } + name, id, err := interpolation.ExtractAccountInformation(account) + if err != nil { + log.WithError(err).WithField("account", account).Error("could not extract account information") + continue + } + accounts = append(accounts, storage.Accounts{ + ID: id, + Name: name, + }) + } + return accounts, nil +} + // GetResources return resource data func (sm *StorageManager) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) { diff --git a/api/storage/elasticsearch/elasticsearch_test.go b/api/storage/elasticsearch/elasticsearch_test.go index d3010bb3..598e2e7d 100644 --- a/api/storage/elasticsearch/elasticsearch_test.go +++ b/api/storage/elasticsearch/elasticsearch_test.go @@ -217,6 +217,64 @@ func TestGetExecutions(t *testing.T) { } } +func TestGetAccounts(t *testing.T) { + + // each different queryLimit will result in a different elasticsearch response. + // 1 - returns valid account data response + // 2 - returns invalid aggregation term query response + // 4 - returns invalid statuscode response + testCases := []struct { + name string + queryLimit int + responseCount int + ErrorMessage error + }{ + {"valid response", 1, 2, nil}, + {"invalid terms", 2, 0, ErrAggregationTermNotFound}, + {"invalid es response", 3, 0, ErrInvalidQuery}, + } + + mockClient, config := testutils.NewESMock(prefixIndexName, true) + + mockClient.Router.HandleFunc("/_search", func(resp http.ResponseWriter, req *http.Request) { + switch testutils.GetPostParams(req) { + case `{"aggregations":{"Accounts":{"terms":{"field":"Data.AccountInformation.keyword","size":1}}},"query":{"match":{"ExecutionID":{"query":"1"}}}}`: + testutils.JSONResponse(resp, http.StatusOK, elastic.SearchResult{Aggregations: map[string]json.RawMessage{ + "Accounts": testutils.LoadResponse("accounts/aggregations/default"), + }}) + case `{"aggregations":{"Accounts":{"terms":{"field":"Data.AccountInformation.keyword","size":2}}},"query":{"match":{"ExecutionID":{"query":"1"}}}}`: + testutils.JSONResponse(resp, http.StatusOK, elastic.SearchResult{Aggregations: map[string]json.RawMessage{ + "invalid-key": testutils.LoadResponse("accounts/aggregations/default"), + }}) + case `{"aggregations":{"Accounts":{"terms":{"field":"Data.AccountInformation.keyword","size":3}}},"query":{"match":{"ExecutionID":{"query":"1"}}}}`: + testutils.JSONResponse(resp, http.StatusBadRequest, elastic.SearchResult{Aggregations: map[string]json.RawMessage{}}) + default: + t.Fatalf("unexpected request params") + } + }) + + es, err := NewStorageManager(config) + if err != nil { + t.Fatalf("unexpected error, got %v expected nil", err) + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + + response, err := es.GetAccounts("1", test.queryLimit) + + if err != test.ErrorMessage { + t.Fatalf("unexpected error, got %v expected %v", err, test.ErrorMessage) + } + + if len(response) != test.responseCount { + t.Fatalf("handler query response: got %d want %d", len(response), test.responseCount) + } + + }) + } +} + func TestGetSummary(t *testing.T) { mockClient, config := testutils.NewESMock(prefixIndexName, true) @@ -225,7 +283,9 @@ func TestGetSummary(t *testing.T) { response := elastic.SearchResult{} - switch testutils.GetPostParams(req) { + var s = testutils.GetPostParams(req) + fmt.Println(s) + switch s { case `{"query":{"bool":{"must":[{"term":{"EventType":"service_status"}},{"term":{"ExecutionID":""}}]}},"size":0}`: response.Hits = &elastic.SearchHits{TotalHits: &elastic.TotalHits{Value: 1}} case `{"query":{"bool":{"must":[{"term":{"EventType":"service_status"}},{"term":{"ExecutionID":""}}]}},"size":1}`: @@ -238,6 +298,9 @@ func TestGetSummary(t *testing.T) { case `{"aggregations":{"sum":{"sum":{"field":"Data.PricePerMonth"}}},"query":{"bool":{"must":[{"match":{"ResourceName":{"minimum_should_match":"100%","query":"aws_resource_name"}}},{"term":{"ExecutionID":""}},{"term":{"EventType":"resource_detected"}}]}},"size":0}`: response.Aggregations = map[string]json.RawMessage{"sum": []byte(`{"value": 36.5}`)} response.Hits = &elastic.SearchHits{TotalHits: &elastic.TotalHits{Value: 1}} + case `{"aggregations":{"accounts":{"aggregations":{"accountSum":{"sum":{"field":"Data.PricePerMonth"}}},"terms":{"field":"Data.AccountID.keyword"}}},"query":{"bool":{"must":[{"match":{"ResourceName":{"minimum_should_match":"100%","query":"aws_resource_name"}}},{"term":{"ExecutionID":""}},{"term":{"EventType":"resource_detected"}}]}},"size":0}`: + response.Aggregations = map[string]json.RawMessage{"accounts": testutils.LoadResponse("summary/aggregations/default")} + response.Hits = &elastic.SearchHits{TotalHits: &elastic.TotalHits{Value: 1}} default: t.Fatalf("unexpected request params") } diff --git a/api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json b/api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json new file mode 100644 index 00000000..ff15d0c8 --- /dev/null +++ b/api/storage/elasticsearch/testutils/responses/accounts/aggregations/default.json @@ -0,0 +1,12 @@ +{ + "buckets" : [ + { + "key" : "Test_123456789", + "doc_count" : 66 + }, + { + "key" : "Test2_12345675", + "doc_count" : 6 + } + ] +} \ No newline at end of file diff --git a/api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json b/api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json new file mode 100644 index 00000000..082107a4 --- /dev/null +++ b/api/storage/elasticsearch/testutils/responses/summary/aggregations/default.json @@ -0,0 +1,11 @@ +{ + "buckets": [ + { + "key": "123456789012", + "doc_count": 1, + "accountSum": { + "value": 36.5 + } + } + ] +} \ No newline at end of file diff --git a/api/storage/structs.go b/api/storage/structs.go index 467b8168..0d453414 100644 --- a/api/storage/structs.go +++ b/api/storage/structs.go @@ -13,6 +13,7 @@ type StorageDescriber interface { Save(data string) bool GetSummary(executionID string, filters map[string]string) (map[string]CollectorsSummary, error) GetExecutions(querylimit int) ([]Executions, error) + GetAccounts(executionID string, querylimit int) ([]Accounts, error) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) GetResourceTrends(resourceType string, filters map[string]string, limit int) ([]ExecutionCost, error) GetExecutionTags(executionID string) (map[string][]string, error) @@ -31,14 +32,20 @@ type ExecutionCost struct { CostSum float64 } +type Accounts struct { + ID string + Name string +} + // CollectorsSummary defines unused resource summary type CollectorsSummary struct { - ResourceName string `json:"ResourceName"` - ResourceCount int64 `json:"ResourceCount"` - TotalSpent float64 `json:"TotalSpent"` - Status int `json:"Status"` - ErrorMessage string `json:"ErrorMessage"` - EventTime int64 `json:"-"` + ResourceName string `json:"ResourceName"` + ResourceCount int64 `json:"ResourceCount"` + TotalSpent float64 `json:"TotalSpent"` + Status int `json:"Status"` + ErrorMessage string `json:"ErrorMessage"` + EventTime int64 `json:"-"` + SpentAccounts map[string]float64 `json:"SpentAccounts"` } type SummaryData struct { diff --git a/api/testutils/storage.go b/api/testutils/storage.go index 7a1a36a2..cc3034b5 100644 --- a/api/testutils/storage.go +++ b/api/testutils/storage.go @@ -65,6 +65,23 @@ func (ms *MockStorage) GetExecutions(queryLimit int) ([]storage.Executions, erro return response, nil } +func (ms *MockStorage) GetAccounts(executionID string, querylimit int) ([]storage.Accounts, error) { + if executionID == "err" { + return nil, errors.New("error") + } + response := []storage.Accounts{ + { + ID: "1234567890", + Name: "Test1", + }, + { + ID: "1234567891", + Name: "Test2", + }, + } + return response, nil +} + func (ms *MockStorage) GetResources(resourceType string, executionID string, filters map[string]string) ([]map[string]interface{}, error) { var response []map[string]interface{} diff --git a/collector/aws/cloudwatch/cloudwatch.go b/collector/aws/cloudwatch/cloudwatch.go index 0317d6e4..8b02fca7 100644 --- a/collector/aws/cloudwatch/cloudwatch.go +++ b/collector/aws/cloudwatch/cloudwatch.go @@ -50,6 +50,10 @@ func (cw *CloudwatchManager) GetMetric(metricInput *awsCloudwatch.GetMetricStati return calculatedMetricValue, metricsResponseValue, err } + if len(metricData.Datapoints) == 0 { + return calculatedMetricValue, metricsResponseValue, errors.New("No Datapoints in this region for this resource.") + } + switch metric.Statistic { case "Average": calculatedMetricValue = cw.AvgDatapoint(metricData) @@ -77,7 +81,6 @@ func (cw *CloudwatchManager) GetMetric(metricInput *awsCloudwatch.GetMetricStati if err != nil { return calculatedMetricValue, metricsResponseValue, err } - return formulaResponse.(float64), metricsResponseValue, nil } diff --git a/collector/aws/common/structs.go b/collector/aws/common/structs.go index ebd7a22b..cb3241c5 100644 --- a/collector/aws/common/structs.go +++ b/collector/aws/common/structs.go @@ -26,6 +26,7 @@ type AWSManager interface { GetCloudWatchClient() *cloudwatch.CloudwatchManager GetPricingClient() *pricing.PricingManager GetRegion() string + GetAccountName() string GetSession() (*session.Session, *aws.Config) GetAccountIdentity() *sts.GetCallerIdentityOutput SetGlobal(resourceName collector.ResourceIdentifier) diff --git a/collector/aws/detector.go b/collector/aws/detector.go index 0450dfa9..857ae7f6 100644 --- a/collector/aws/detector.go +++ b/collector/aws/detector.go @@ -21,6 +21,7 @@ type DetectorDescriptor interface { GetCloudWatchClient() *cloudwatch.CloudwatchManager GetPricingClient() *pricing.PricingManager GetRegion() string + GetAccountName() string GetSession() (*session.Session, *awsClient.Config) GetAccountIdentity() *sts.GetCallerIdentityOutput } @@ -38,12 +39,20 @@ type DetectorManager struct { session *session.Session awsConfig *awsClient.Config accountIdentity *sts.GetCallerIdentityOutput + accountName string region string global map[string]struct{} } // NewDetectorManager create new instance of detector manager -func NewDetectorManager(awsAuth AuthDescriptor, collector collector.CollectorDescriber, account config.AWSAccount, stsManager *STSManager, global map[string]struct{}, region string) *DetectorManager { +func NewDetectorManager( + awsAuth AuthDescriptor, + collector collector.CollectorDescriber, + account config.AWSAccount, + stsManager *STSManager, + global map[string]struct{}, + region string, +) *DetectorManager { priceSession, _ := awsAuth.Login(defaultRegionPrice) pricingManager := pricing.NewPricingManager(awsPricing.New(priceSession), defaultRegionPrice) @@ -60,6 +69,7 @@ func NewDetectorManager(awsAuth AuthDescriptor, collector collector.CollectorDes session: regionSession, awsConfig: regionConfig, accountIdentity: callerIdentityOutput, + accountName: account.Name, global: global, } } @@ -89,6 +99,11 @@ func (dm *DetectorManager) GetRegion() string { return dm.region } +// GetAccountName returns the account name +func (dm *DetectorManager) GetAccountName() string { + return dm.accountName +} + // GetSession return the aws session func (dm *DetectorManager) GetSession() (*session.Session, *awsClient.Config) { return dm.session, dm.awsConfig diff --git a/collector/aws/resources/apigateway.go b/collector/aws/resources/apigateway.go index 0157e07d..ea79eaec 100644 --- a/collector/aws/resources/apigateway.go +++ b/collector/aws/resources/apigateway.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -36,6 +37,7 @@ type DetectedAPIGateway struct { Name string LaunchTime time.Time Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -71,7 +73,10 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "apigateway", }).Info("starting to analyze resource") - ag.awsManager.GetCollector().CollectStart(ag.Name) + ag.awsManager.GetCollector().CollectStart(ag.Name, collector.AccountSpecifiedFields{ + AccountID: *ag.awsManager.GetAccountIdentity().Account, + AccountName: ag.awsManager.GetAccountName(), + }) detectAPIGateway := []DetectedAPIGateway{} apigateways, err := ag.getRestApis(nil, nil) @@ -140,13 +145,25 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:apigateway:" + ag.awsManager.GetRegion() + "::/restapis/" + *api.Id + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + detect := DetectedAPIGateway{ Region: ag.awsManager.GetRegion(), Metric: metric.Description, - ResourceID: *api.Id, + ResourceID: Arn, Name: *api.Name, LaunchTime: *api.CreatedDate, Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ag.awsManager.GetAccountIdentity().Account, + AccountName: ag.awsManager.GetAccountName(), + }, } ag.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -160,7 +177,10 @@ func (ag *APIGatewayManager) Detect(metrics []config.MetricConfig) (interface{}, } } - ag.awsManager.GetCollector().CollectFinish(ag.Name) + ag.awsManager.GetCollector().CollectFinish(ag.Name, collector.AccountSpecifiedFields{ + AccountID: *ag.awsManager.GetAccountIdentity().Account, + AccountName: ag.awsManager.GetAccountName(), + }) return detectAPIGateway, nil } diff --git a/collector/aws/resources/docdb.go b/collector/aws/resources/docdb.go index 5753c94f..1929dde3 100644 --- a/collector/aws/resources/docdb.go +++ b/collector/aws/resources/docdb.go @@ -39,6 +39,7 @@ type DetectedDocumentDB struct { MultiAZ bool Engine string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -74,7 +75,10 @@ func (dd *DocumentDBManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "documentDB", }).Info("starting to analyze resource") - dd.awsManager.GetCollector().CollectStart(dd.Name) + dd.awsManager.GetCollector().CollectStart(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) detectedDocDB := []DetectedDocumentDB{} instances, err := dd.describeInstances(nil, nil) @@ -162,6 +166,10 @@ func (dd *DocumentDBManager) Detect(metrics []config.MetricConfig) (interface{}, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }, } dd.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -176,7 +184,10 @@ func (dd *DocumentDBManager) Detect(metrics []config.MetricConfig) (interface{}, } - dd.awsManager.GetCollector().CollectFinish(dd.Name) + dd.awsManager.GetCollector().CollectFinish(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) return detectedDocDB, nil diff --git a/collector/aws/resources/dynamodb.go b/collector/aws/resources/dynamodb.go index 6c109980..084b1260 100644 --- a/collector/aws/resources/dynamodb.go +++ b/collector/aws/resources/dynamodb.go @@ -42,6 +42,7 @@ type DetectedAWSDynamoDB struct { Metric string Name string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -78,7 +79,10 @@ func (dd *DynamoDBManager) Detect(metrics []config.MetricConfig) (interface{}, e "resource": "dynamoDB", }).Info("starting to analyze resource") - dd.awsManager.GetCollector().CollectStart(dd.Name) + dd.awsManager.GetCollector().CollectStart(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) detectedTables := []DetectedAWSDynamoDB{} tables, err := dd.describeTables(nil, nil) @@ -199,6 +203,10 @@ func (dd *DynamoDBManager) Detect(metrics []config.MetricConfig) (interface{}, e PricePerMonth: pricePerMonth, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }, } dd.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -212,7 +220,10 @@ func (dd *DynamoDBManager) Detect(metrics []config.MetricConfig) (interface{}, e } } - dd.awsManager.GetCollector().CollectFinish(dd.Name) + dd.awsManager.GetCollector().CollectFinish(dd.Name, collector.AccountSpecifiedFields{ + AccountID: *dd.awsManager.GetAccountIdentity().Account, + AccountName: dd.awsManager.GetAccountName(), + }) return detectedTables, nil diff --git a/collector/aws/resources/ec2.go b/collector/aws/resources/ec2.go index 1bba1697..e4ba0d13 100644 --- a/collector/aws/resources/ec2.go +++ b/collector/aws/resources/ec2.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "strings" "time" @@ -38,6 +39,7 @@ type DetectedEC2 struct { Name string InstanceType string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -73,7 +75,10 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "ec2_instances", }).Info("starting to analyze resource") - ec.awsManager.GetCollector().CollectStart(ec.Name) + ec.awsManager.GetCollector().CollectStart(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) detectedEC2 := []DetectedEC2{} @@ -152,18 +157,30 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) } } + Arn := "arn:aws:ec2:" + ec.awsManager.GetRegion() + ":" + *ec.awsManager.GetAccountIdentity().Account + ":instance/" + *instance.InstanceId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + ec2 := DetectedEC2{ Region: ec.awsManager.GetRegion(), Metric: metric.Description, Name: name, InstanceType: *instance.InstanceType, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *instance.InstanceId, + ResourceID: Arn, LaunchTime: *instance.LaunchTime, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }, } ec.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -178,7 +195,10 @@ func (ec *EC2Manager) Detect(metrics []config.MetricConfig) (interface{}, error) } } - ec.awsManager.GetCollector().CollectFinish(ec.Name) + ec.awsManager.GetCollector().CollectFinish(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) return detectedEC2, nil diff --git a/collector/aws/resources/ec2volumes.go b/collector/aws/resources/ec2volumes.go index ac5e0fa3..cdf1908a 100644 --- a/collector/aws/resources/ec2volumes.go +++ b/collector/aws/resources/ec2volumes.go @@ -6,6 +6,7 @@ import ( "finala/collector/aws/common" "finala/collector/aws/register" "finala/collector/config" + "github.com/aws/aws-sdk-go/aws/arn" awsClient "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -36,6 +37,7 @@ type DetectedAWSEC2Volume struct { Size int64 PricePerMonth float64 Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -75,7 +77,10 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "ec2_volume", }).Info("starting to analyze resource") - ev.awsManager.GetCollector().CollectStart(ev.Name) + ev.awsManager.GetCollector().CollectStart(ev.Name, collector.AccountSpecifiedFields{ + AccountID: *ev.awsManager.GetAccountIdentity().Account, + AccountName: ev.awsManager.GetAccountName(), + }) detected := []DetectedAWSEC2Volume{} volumes, err := ev.describe(nil, nil) @@ -115,15 +120,27 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:ec2:" + ev.awsManager.GetRegion() + ":" + *ev.awsManager.GetAccountIdentity().Account + ":volume/" + *vol.VolumeId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + volumeSize := *vol.Size dEBS := DetectedAWSEC2Volume{ Region: ev.awsManager.GetRegion(), Metric: metric.Description, - ResourceID: *vol.VolumeId, + ResourceID: Arn, Type: *vol.VolumeType, Size: volumeSize, PricePerMonth: ev.getCalculatedPrice(vol, price), Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ev.awsManager.GetAccountIdentity().Account, + AccountName: ev.awsManager.GetAccountName(), + }, } ev.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -135,7 +152,10 @@ func (ev *EC2VolumeManager) Detect(metrics []config.MetricConfig) (interface{}, } - ev.awsManager.GetCollector().CollectFinish(ev.Name) + ev.awsManager.GetCollector().CollectFinish(ev.Name, collector.AccountSpecifiedFields{ + AccountID: *ev.awsManager.GetAccountIdentity().Account, + AccountName: ev.awsManager.GetAccountName(), + }) return detected, nil diff --git a/collector/aws/resources/elasticache.go b/collector/aws/resources/elasticache.go index 473432ab..06f42fe8 100644 --- a/collector/aws/resources/elasticache.go +++ b/collector/aws/resources/elasticache.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -39,6 +40,7 @@ type DetectedElasticache struct { CacheNodeType string CacheNodes int collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -74,7 +76,10 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} "resource": "elasticache", }).Info("starting to analyze resource") - ec.awsManager.GetCollector().CollectStart(ec.Name) + ec.awsManager.GetCollector().CollectStart(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) detectedelasticache := []DetectedElasticache{} @@ -150,6 +155,14 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} } } + Arn := "arn:aws:elasticache:" + ec.awsManager.GetRegion() + ":" + *ec.awsManager.GetAccountIdentity().Account + ":cluster:" + *instance.CacheClusterId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + es := DetectedElasticache{ Region: ec.awsManager.GetRegion(), Metric: metric.Description, @@ -158,11 +171,15 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} CacheNodes: len(instance.CacheNodes), PriceDetectedFields: collector.PriceDetectedFields{ LaunchTime: *instance.CacheClusterCreateTime, - ResourceID: *instance.CacheClusterId, + ResourceID: Arn, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }, } ec.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -175,7 +192,10 @@ func (ec *ElasticacheManager) Detect(metrics []config.MetricConfig) (interface{} } } - ec.awsManager.GetCollector().CollectFinish(ec.Name) + ec.awsManager.GetCollector().CollectFinish(ec.Name, collector.AccountSpecifiedFields{ + AccountID: *ec.awsManager.GetAccountIdentity().Account, + AccountName: ec.awsManager.GetAccountName(), + }) return detectedelasticache, nil } diff --git a/collector/aws/resources/elasticips.go b/collector/aws/resources/elasticips.go index b22b94fd..dbc7d8a0 100644 --- a/collector/aws/resources/elasticips.go +++ b/collector/aws/resources/elasticips.go @@ -6,6 +6,7 @@ import ( "finala/collector/aws/common" "finala/collector/aws/register" "finala/collector/config" + "github.com/aws/aws-sdk-go/aws/arn" awsClient "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -29,12 +30,11 @@ type ElasticIPManager struct { // DetectedElasticIP defines the detected AWS elastic ip type DetectedElasticIP struct { - Region string - Metric string - IP string - PricePerHour float64 - PricePerMonth float64 - Tag map[string]string + Region string + Metric string + IP string + collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -72,7 +72,10 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "elastic ips", }).Info("starting to analyze resource") - ei.awsManager.GetCollector().CollectStart(ei.Name) + ei.awsManager.GetCollector().CollectStart(ei.Name, collector.AccountSpecifiedFields{ + AccountID: *ei.awsManager.GetAccountIdentity().Account, + AccountName: ei.awsManager.GetAccountName(), + }) elasticIPs := []DetectedElasticIP{} @@ -108,13 +111,28 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:ec2:" + ei.awsManager.GetRegion() + ":" + *ei.awsManager.GetAccountIdentity().Account + ":elastic-ip/" + *ip.AllocationId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + eIP := DetectedElasticIP{ - Region: ei.awsManager.GetRegion(), - Metric: metric.Description, - IP: *ip.PublicIp, - PricePerHour: price, - PricePerMonth: price * collector.TotalMonthHours, - Tag: tagsData, + Region: ei.awsManager.GetRegion(), + Metric: metric.Description, + IP: *ip.PublicIp, + PriceDetectedFields: collector.PriceDetectedFields{ + ResourceID: Arn, + PricePerHour: price, + PricePerMonth: price * collector.TotalMonthHours, + Tag: tagsData, + }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ei.awsManager.GetAccountIdentity().Account, + AccountName: ei.awsManager.GetAccountName(), + }, } ei.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -127,7 +145,10 @@ func (ei *ElasticIPManager) Detect(metrics []config.MetricConfig) (interface{}, } } - ei.awsManager.GetCollector().CollectFinish(ei.Name) + ei.awsManager.GetCollector().CollectFinish(ei.Name, collector.AccountSpecifiedFields{ + AccountID: *ei.awsManager.GetAccountIdentity().Account, + AccountName: ei.awsManager.GetAccountName(), + }) return elasticIPs, nil diff --git a/collector/aws/resources/elasticips_test.go b/collector/aws/resources/elasticips_test.go index cce26355..56e14de5 100644 --- a/collector/aws/resources/elasticips_test.go +++ b/collector/aws/resources/elasticips_test.go @@ -21,12 +21,15 @@ var defaultAddressesMock = ec2.DescribeAddressesOutput{ AssociationId: awsClient.String("foo-00000"), InstanceId: awsClient.String("i-00000"), NetworkInterfaceId: awsClient.String("00000"), + AllocationId: awsClient.String("aws_eip.example.id"), }, { - PublicIp: awsClient.String("80.80.80.81"), + PublicIp: awsClient.String("80.80.80.81"), + AllocationId: awsClient.String("aws_eip.example.id"), }, { - PublicIp: awsClient.String("80.80.80.82"), + PublicIp: awsClient.String("80.80.80.82"), + AllocationId: awsClient.String("aws_eip.example.id"), }, }, } diff --git a/collector/aws/resources/elasticsearch.go b/collector/aws/resources/elasticsearch.go index 03e4e797..13a7fb38 100644 --- a/collector/aws/resources/elasticsearch.go +++ b/collector/aws/resources/elasticsearch.go @@ -46,6 +46,7 @@ type DetectedElasticSearch struct { InstanceType string InstanceCount int64 collector.PriceDetectedFields + collector.AccountSpecifiedFields } // elasticSearchVolumeType will hold the available volume types for ESCluster EBS @@ -88,7 +89,10 @@ func (esm *ElasticSearchManager) Detect(metrics []config.MetricConfig) (interfac "resource": "elasticsearch", }).Info("analyzing resource") - esm.awsManager.GetCollector().CollectStart(esm.Name) + esm.awsManager.GetCollector().CollectStart(esm.Name, collector.AccountSpecifiedFields{ + AccountID: *esm.awsManager.GetAccountIdentity().Account, + AccountName: esm.awsManager.GetAccountName(), + }) detectedElasticSearchClusters := []DetectedElasticSearch{} @@ -220,6 +224,10 @@ func (esm *ElasticSearchManager) Detect(metrics []config.MetricConfig) (interfac PricePerMonth: hourlyClusterPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *esm.awsManager.GetAccountIdentity().Account, + AccountName: esm.awsManager.GetAccountName(), + }, } esm.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -232,7 +240,10 @@ func (esm *ElasticSearchManager) Detect(metrics []config.MetricConfig) (interfac } } - esm.awsManager.GetCollector().CollectFinish(esm.Name) + esm.awsManager.GetCollector().CollectFinish(esm.Name, collector.AccountSpecifiedFields{ + AccountID: *esm.awsManager.GetAccountIdentity().Account, + AccountName: esm.awsManager.GetAccountName(), + }) return detectedElasticSearchClusters, nil } diff --git a/collector/aws/resources/elb.go b/collector/aws/resources/elb.go index 4ef6cfec..e64c20c8 100644 --- a/collector/aws/resources/elb.go +++ b/collector/aws/resources/elb.go @@ -8,6 +8,7 @@ import ( "finala/collector/config" "finala/expression" "fmt" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -37,6 +38,7 @@ type DetectedELB struct { Metric string Region string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -72,7 +74,10 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "elb", }).Info("starting to analyze resource") - el.awsManager.GetCollector().CollectStart(el.Name) + el.awsManager.GetCollector().CollectStart(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) detectedELB := []DetectedELB{} @@ -166,16 +171,28 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) } } + Arn := "arn:aws:elasticloadbalancing:" + el.awsManager.GetRegion() + ":" + *el.awsManager.GetAccountIdentity().Account + ":loadbalancer/" + *instance.LoadBalancerName + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + elb := DetectedELB{ Region: el.awsManager.GetRegion(), Metric: metric.Description, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *instance.LoadBalancerName, + ResourceID: Arn, LaunchTime: *instance.CreatedTime, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }, } el.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -190,7 +207,10 @@ func (el *ELBManager) Detect(metrics []config.MetricConfig) (interface{}, error) } } - el.awsManager.GetCollector().CollectFinish(el.Name) + el.awsManager.GetCollector().CollectFinish(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) return detectedELB, nil diff --git a/collector/aws/resources/elbv2.go b/collector/aws/resources/elbv2.go index c74ed42e..ec6eac89 100644 --- a/collector/aws/resources/elbv2.go +++ b/collector/aws/resources/elbv2.go @@ -38,6 +38,7 @@ type DetectedELBV2 struct { Region string Type string collector.PriceDetectedFields + collector.AccountSpecifiedFields } // loadBalancerConfig defines loadbalancer's configuration of metrics and pricing @@ -103,7 +104,10 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro "resource": "elb_v2", }).Info("starting to analyze resource") - el.awsManager.GetCollector().CollectStart(el.Name) + el.awsManager.GetCollector().CollectStart(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) detectedELBV2 := []DetectedELBV2{} @@ -213,12 +217,16 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro Metric: metric.Description, Type: *instance.Type, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *instance.LoadBalancerName, + ResourceID: *instance.LoadBalancerArn, LaunchTime: *instance.CreatedTime, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }, } el.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -231,7 +239,10 @@ func (el *ELBV2Manager) Detect(metrics []config.MetricConfig) (interface{}, erro } } - el.awsManager.GetCollector().CollectFinish(el.Name) + el.awsManager.GetCollector().CollectFinish(el.Name, collector.AccountSpecifiedFields{ + AccountID: *el.awsManager.GetAccountIdentity().Account, + AccountName: el.awsManager.GetAccountName(), + }) return detectedELBV2, nil diff --git a/collector/aws/resources/iam.go b/collector/aws/resources/iam.go index 4aa8ca20..ad5ac99d 100644 --- a/collector/aws/resources/iam.go +++ b/collector/aws/resources/iam.go @@ -34,6 +34,8 @@ type DetectedAWSLastActivity struct { AccessKey string LastUsedDate time.Time LastActivity string + ResourceID string + collector.AccountSpecifiedFields } func init() { @@ -75,7 +77,10 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "iam", }).Info("starting to analyze resource") - im.awsManager.GetCollector().CollectStart(im.Name) + im.awsManager.GetCollector().CollectStart(im.Name, collector.AccountSpecifiedFields{ + AccountID: *im.awsManager.GetAccountIdentity().Account, + AccountName: im.awsManager.GetAccountName(), + }) detected := []DetectedAWSLastActivity{} @@ -133,6 +138,11 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) AccessKey: *accessKeyData.AccessKeyId, LastUsedDate: lastUsedDate, LastActivity: lastActivity, + ResourceID: *user.Arn, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *im.awsManager.GetAccountIdentity().Account, + AccountName: im.awsManager.GetAccountName(), + }, } im.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -146,7 +156,10 @@ func (im *IAMManager) Detect(metrics []config.MetricConfig) (interface{}, error) } } - im.awsManager.GetCollector().CollectFinish(im.Name) + im.awsManager.GetCollector().CollectFinish(im.Name, collector.AccountSpecifiedFields{ + AccountID: *im.awsManager.GetAccountIdentity().Account, + AccountName: im.awsManager.GetAccountName(), + }) return detected, nil } diff --git a/collector/aws/resources/iam_test.go b/collector/aws/resources/iam_test.go index cb6a6813..c1616e6b 100644 --- a/collector/aws/resources/iam_test.go +++ b/collector/aws/resources/iam_test.go @@ -16,9 +16,17 @@ import ( var defaultUsersMock = iam.ListUsersOutput{ Users: []*iam.User{ - {UserName: awsClient.String("foo")}, - {UserName: awsClient.String("foo2")}, - {UserName: awsClient.String("test")}, + { + UserName: awsClient.String("foo"), + Arn: awsClient.String("arn:aws:iam::123456789012:user/foo")}, + { + UserName: awsClient.String("foo2"), + Arn: awsClient.String("arn:aws:iam::123456789012:user/foo2"), + }, + { + UserName: awsClient.String("test"), + Arn: awsClient.String("arn:aws:iam::123456789012:user/test"), + }, }, } diff --git a/collector/aws/resources/kinesis.go b/collector/aws/resources/kinesis.go index f1cd3d1c..16c6e571 100644 --- a/collector/aws/resources/kinesis.go +++ b/collector/aws/resources/kinesis.go @@ -38,6 +38,7 @@ type DetectedKinesis struct { Metric string Region string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -74,7 +75,10 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er "resource": "kinesis", }).Info("analyzing resource") - km.awsManager.GetCollector().CollectStart(km.Name) + km.awsManager.GetCollector().CollectStart(km.Name, collector.AccountSpecifiedFields{ + AccountID: *km.awsManager.GetAccountIdentity().Account, + AccountName: km.awsManager.GetAccountName(), + }) streams, err := km.describeStreams(nil, nil) if err != nil { @@ -187,12 +191,16 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er Region: km.awsManager.GetRegion(), Metric: metric.Description, PriceDetectedFields: collector.PriceDetectedFields{ - ResourceID: *stream.StreamName, + ResourceID: *stream.StreamARN, LaunchTime: *stream.StreamCreationTimestamp, PricePerHour: totalShardsPerHourPrice, PricePerMonth: totalShardsPerHourPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *km.awsManager.GetAccountIdentity().Account, + AccountName: km.awsManager.GetAccountName(), + }, } km.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -204,7 +212,10 @@ func (km *KinesisManager) Detect(metrics []config.MetricConfig) (interface{}, er } } } - km.awsManager.GetCollector().CollectFinish(km.Name) + km.awsManager.GetCollector().CollectFinish(km.Name, collector.AccountSpecifiedFields{ + AccountID: *km.awsManager.GetAccountIdentity().Account, + AccountName: km.awsManager.GetAccountName(), + }) return detectedStreams, nil } diff --git a/collector/aws/resources/lambda.go b/collector/aws/resources/lambda.go index e6e2569f..4de76190 100644 --- a/collector/aws/resources/lambda.go +++ b/collector/aws/resources/lambda.go @@ -36,6 +36,7 @@ type DetectedAWSLambda struct { ResourceID string Name string Tag map[string]string + collector.AccountSpecifiedFields } func init() { @@ -70,7 +71,10 @@ func (lm *LambdaManager) Detect(metrics []config.MetricConfig) (interface{}, err "resource": "lambda", }).Info("starting to analyze resource") - lm.awsManager.GetCollector().CollectStart(lm.Name) + lm.awsManager.GetCollector().CollectStart(lm.Name, collector.AccountSpecifiedFields{ + AccountID: *lm.awsManager.GetAccountIdentity().Account, + AccountName: lm.awsManager.GetAccountName(), + }) detected := []DetectedAWSLambda{} functions, err := lm.describe(nil, nil) @@ -149,6 +153,10 @@ func (lm *LambdaManager) Detect(metrics []config.MetricConfig) (interface{}, err ResourceID: *fun.FunctionArn, Name: *fun.FunctionName, Tag: tagsData, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *lm.awsManager.GetAccountIdentity().Account, + AccountName: lm.awsManager.GetAccountName(), + }, } lm.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -161,7 +169,10 @@ func (lm *LambdaManager) Detect(metrics []config.MetricConfig) (interface{}, err } } - lm.awsManager.GetCollector().CollectFinish(lm.Name) + lm.awsManager.GetCollector().CollectFinish(lm.Name, collector.AccountSpecifiedFields{ + AccountID: *lm.awsManager.GetAccountIdentity().Account, + AccountName: lm.awsManager.GetAccountName(), + }) return detected, nil } diff --git a/collector/aws/resources/natgateway.go b/collector/aws/resources/natgateway.go index a4f788e5..52ceac21 100644 --- a/collector/aws/resources/natgateway.go +++ b/collector/aws/resources/natgateway.go @@ -8,6 +8,7 @@ import ( "finala/collector/config" "finala/expression" "fmt" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -39,6 +40,7 @@ type DetectedNATGateway struct { SubnetID string VPCID string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -74,7 +76,10 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} "resource": "natgateway", }).Info("analyzing resource") - ngw.awsManager.GetCollector().CollectStart(ngw.Name) + ngw.awsManager.GetCollector().CollectStart(ngw.Name, collector.AccountSpecifiedFields{ + AccountID: *ngw.awsManager.GetAccountIdentity().Account, + AccountName: ngw.awsManager.GetAccountName(), + }) DetectedNATGateways := []DetectedNATGateway{} @@ -166,6 +171,14 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} } } + Arn := "arn:aws:ec2:" + ngw.awsManager.GetRegion() + ":" + *ngw.awsManager.GetAccountIdentity().Account + ":natgateway/" + *natgateway.NatGatewayId + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + natGateway := DetectedNATGateway{ Region: ngw.awsManager.GetRegion(), Metric: metric.Description, @@ -173,11 +186,15 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} VPCID: *natgateway.VpcId, PriceDetectedFields: collector.PriceDetectedFields{ LaunchTime: *natgateway.CreateTime, - ResourceID: *natgateway.NatGatewayId, + ResourceID: Arn, PricePerHour: price, PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *ngw.awsManager.GetAccountIdentity().Account, + AccountName: ngw.awsManager.GetAccountName(), + }, } ngw.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -190,7 +207,10 @@ func (ngw *NatGatewayManager) Detect(metrics []config.MetricConfig) (interface{} } } - ngw.awsManager.GetCollector().CollectFinish(ngw.Name) + ngw.awsManager.GetCollector().CollectFinish(ngw.Name, collector.AccountSpecifiedFields{ + AccountID: *ngw.awsManager.GetAccountIdentity().Account, + AccountName: ngw.awsManager.GetAccountName(), + }) return DetectedNATGateways, nil } diff --git a/collector/aws/resources/neptune.go b/collector/aws/resources/neptune.go index 7a22ad3f..8a46eebc 100644 --- a/collector/aws/resources/neptune.go +++ b/collector/aws/resources/neptune.go @@ -39,6 +39,7 @@ type DetectedAWSNeptune struct { MultiAZ bool Engine string collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -74,7 +75,10 @@ func (np *NeptuneManager) Detect(metrics []config.MetricConfig) (interface{}, er "resource": "neptune", }).Info("starting to analyze resource") - np.awsManager.GetCollector().CollectStart(np.Name) + np.awsManager.GetCollector().CollectStart(np.Name, collector.AccountSpecifiedFields{ + AccountID: *np.awsManager.GetAccountIdentity().Account, + AccountName: np.awsManager.GetAccountName(), + }) detected := []DetectedAWSNeptune{} instances, err := np.describeInstances(nil, nil) @@ -163,6 +167,10 @@ func (np *NeptuneManager) Detect(metrics []config.MetricConfig) (interface{}, er PricePerMonth: price * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *np.awsManager.GetAccountIdentity().Account, + AccountName: np.awsManager.GetAccountName(), + }, } np.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -173,7 +181,10 @@ func (np *NeptuneManager) Detect(metrics []config.MetricConfig) (interface{}, er } } } - np.awsManager.GetCollector().CollectFinish(np.Name) + np.awsManager.GetCollector().CollectFinish(np.Name, collector.AccountSpecifiedFields{ + AccountID: *np.awsManager.GetAccountIdentity().Account, + AccountName: np.awsManager.GetAccountName(), + }) return detected, nil } diff --git a/collector/aws/resources/rds.go b/collector/aws/resources/rds.go index ba2d5d67..179bebfa 100644 --- a/collector/aws/resources/rds.go +++ b/collector/aws/resources/rds.go @@ -43,6 +43,7 @@ type DetectedAWSRDS struct { MultiAZ bool Engine string collector.PriceDetectedFields + collector.AccountSpecifiedFields } // RDSVolumeType will hold the available volume types for RDS types @@ -86,7 +87,10 @@ func (r *RDSManager) Detect(metrics []config.MetricConfig) (interface{}, error) "resource": "rds", }).Info("starting to analyze resource") - r.awsManager.GetCollector().CollectStart(r.Name) + r.awsManager.GetCollector().CollectStart(r.Name, collector.AccountSpecifiedFields{ + AccountID: *r.awsManager.GetAccountIdentity().Account, + AccountName: r.awsManager.GetAccountName(), + }) detected := []DetectedAWSRDS{} @@ -206,6 +210,10 @@ func (r *RDSManager) Detect(metrics []config.MetricConfig) (interface{}, error) PricePerMonth: totalHourlyPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *r.awsManager.GetAccountIdentity().Account, + AccountName: r.awsManager.GetAccountName(), + }, } r.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -219,7 +227,10 @@ func (r *RDSManager) Detect(metrics []config.MetricConfig) (interface{}, error) } - r.awsManager.GetCollector().CollectFinish(r.Name) + r.awsManager.GetCollector().CollectFinish(r.Name, collector.AccountSpecifiedFields{ + AccountID: *r.awsManager.GetAccountIdentity().Account, + AccountName: r.awsManager.GetAccountName(), + }) return detected, nil diff --git a/collector/aws/resources/redshift.go b/collector/aws/resources/redshift.go index 7f660d6e..e791fac3 100644 --- a/collector/aws/resources/redshift.go +++ b/collector/aws/resources/redshift.go @@ -7,6 +7,7 @@ import ( "finala/collector/aws/register" "finala/collector/config" "finala/expression" + "github.com/aws/aws-sdk-go/aws/arn" "time" awsClient "github.com/aws/aws-sdk-go/aws" @@ -38,6 +39,7 @@ type DetectedRedShift struct { NodeType string NumberOfNodes int64 collector.PriceDetectedFields + collector.AccountSpecifiedFields } func init() { @@ -73,7 +75,10 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, "resource": "redshift", }).Info("analyzing resource") - rdm.awsManager.GetCollector().CollectStart(rdm.Name) + rdm.awsManager.GetCollector().CollectStart(rdm.Name, collector.AccountSpecifiedFields{ + AccountID: *rdm.awsManager.GetAccountIdentity().Account, + AccountName: rdm.awsManager.GetAccountName(), + }) detectedredshiftClusters := []DetectedRedShift{} @@ -147,6 +152,14 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, } } + Arn := "arn:aws:redshift:" + rdm.awsManager.GetRegion() + ":" + *rdm.awsManager.GetAccountIdentity().Account + ":cluster:" + *cluster.ClusterIdentifier + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + redshift := DetectedRedShift{ Region: rdm.awsManager.GetRegion(), Metric: metric.Description, @@ -154,11 +167,15 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, NumberOfNodes: *cluster.NumberOfNodes, PriceDetectedFields: collector.PriceDetectedFields{ LaunchTime: *cluster.ClusterCreateTime, - ResourceID: *cluster.ClusterIdentifier, + ResourceID: Arn, PricePerHour: clusterPrice, PricePerMonth: clusterPrice * collector.TotalMonthHours, Tag: tagsData, }, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *rdm.awsManager.GetAccountIdentity().Account, + AccountName: rdm.awsManager.GetAccountName(), + }, } rdm.awsManager.GetCollector().AddResource(collector.EventCollector{ @@ -171,7 +188,10 @@ func (rdm *RedShiftManager) Detect(metrics []config.MetricConfig) (interface{}, } } - rdm.awsManager.GetCollector().CollectFinish(rdm.Name) + rdm.awsManager.GetCollector().CollectFinish(rdm.Name, collector.AccountSpecifiedFields{ + AccountID: *rdm.awsManager.GetAccountIdentity().Account, + AccountName: rdm.awsManager.GetAccountName(), + }) return detectedredshiftClusters, nil } diff --git a/collector/aws/resources/s3.go b/collector/aws/resources/s3.go new file mode 100644 index 00000000..4ca5dd92 --- /dev/null +++ b/collector/aws/resources/s3.go @@ -0,0 +1,196 @@ +package resources + +import ( + "errors" + "finala/collector" + "finala/collector/aws/common" + "finala/collector/aws/register" + "finala/collector/config" + "finala/expression" + awsClient "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/arn" + awsCloudwatch "github.com/aws/aws-sdk-go/service/cloudwatch" + "github.com/aws/aws-sdk-go/service/s3" + log "github.com/sirupsen/logrus" + "time" +) + +// S3ClientDescriptor is an interface defining the aws s3 client +type S3ClientDescriptor interface { + ListBuckets(input *s3.ListBucketsInput) (*s3.ListBucketsOutput, error) +} + +// S3Manager describes S3 struct +type S3Manager struct { + client S3ClientDescriptor + awsManager common.AWSManager + namespace string + servicePricingCode string + Name collector.ResourceIdentifier +} + +// DetectedS3 define the detected AWS S3 +type DetectedS3 struct { + Region string + Metric string + Name string + ResourceID string + LaunchTime time.Time + collector.AccountSpecifiedFields +} + +func init() { + register.Registry("s3", NewS3Manager) +} + +// NewS3Manager implements AWS GO SDK +func NewS3Manager(awsManager common.AWSManager, client interface{}) (common.ResourceDetection, error) { + + if client == nil { + client = s3.New(awsManager.GetSession()) + } + + s3Client, ok := client.(S3ClientDescriptor) + if !ok { + return nil, errors.New("invalid s3 client") + } + + return &S3Manager{ + client: s3Client, + awsManager: awsManager, + namespace: "AWS/S3", + servicePricingCode: "AmazonS3", + Name: awsManager.GetResourceIdentifier("s3"), + }, nil + +} + +// Detect S3 buckets is under utilized +func (s *S3Manager) Detect(metrics []config.MetricConfig) (interface{}, error) { + + log.WithFields(log.Fields{ + "region": s.awsManager.GetRegion(), + "resource": "s3_buckets", + }).Info("starting to analyze resource") + + s.awsManager.GetCollector().CollectStart(s.Name, collector.AccountSpecifiedFields{ + AccountID: *s.awsManager.GetAccountIdentity().Account, + AccountName: s.awsManager.GetAccountName(), + }) + + detectedS3 := []DetectedS3{} + + buckets, err := s.listBuckets(nil) + if err != nil { + s.awsManager.GetCollector().CollectError(s.Name, err) + return detectedS3, err + } + now := time.Now() + + for _, bucket := range buckets { + log.WithField("bucket_name", *bucket.Name).Debug("checking s3 bucket") + + for _, metric := range metrics { + + period := int64(metric.Period.Seconds()) + metricEndTime := now.Add(time.Duration(-metric.StartTime)) + metricInput := awsCloudwatch.GetMetricStatisticsInput{ + Namespace: &s.namespace, + MetricName: &metric.Description, + Period: &period, + StartTime: &metricEndTime, + EndTime: &now, + Dimensions: []*awsCloudwatch.Dimension{ + { + Name: awsClient.String("BucketName"), + Value: bucket.Name, + }, + { + Name: awsClient.String("FilterId"), + Value: awsClient.String("EntireBucket"), + }, + }, + } + + formulaValue, _, err := s.awsManager.GetCloudWatchClient().GetMetric(&metricInput, metric) + if err != nil { + log.WithError(err).WithFields(log.Fields{ + "bucket_name": *bucket.Name, + "metric_name": metric.Description, + }).Error("Could not get cloudwatch metric data") + continue + } + + expression, err := expression.BoolExpression(formulaValue, metric.Constraint.Value, metric.Constraint.Operator) + if err != nil || formulaValue == float64(-1) { + log.Info("formel == -1") + continue + } + + if expression { + + log.WithFields(log.Fields{ + "metric_name": metric.Description, + "constraint_operator": metric.Constraint.Operator, + "constraint_Value": metric.Constraint.Value, + "formula_value": formulaValue, + "bucket_name": *bucket.Name, + "region": s.awsManager.GetRegion(), + }).Info("EC2 instance detected as unutilized resource") + + Arn := "arn:aws:s3:::" + *bucket.Name + + if !arn.IsARN(Arn) { + log.WithFields(log.Fields{ + "arn": Arn, + }).Error("is not an arn") + } + + s3 := DetectedS3{ + Region: s.awsManager.GetRegion(), + Metric: metric.Description, + Name: *bucket.Name, + ResourceID: Arn, + LaunchTime: *bucket.CreationDate, + AccountSpecifiedFields: collector.AccountSpecifiedFields{ + AccountID: *s.awsManager.GetAccountIdentity().Account, + AccountName: s.awsManager.GetAccountName(), + }, + } + + s.awsManager.GetCollector().AddResource(collector.EventCollector{ + ResourceName: s.Name, + Data: s3, + }) + + detectedS3 = append(detectedS3, s3) + } + } + } + + s.awsManager.GetCollector().CollectFinish(s.Name, collector.AccountSpecifiedFields{ + AccountID: *s.awsManager.GetAccountIdentity().Account, + AccountName: s.awsManager.GetAccountName(), + }) + + return detectedS3, nil +} + +func (s *S3Manager) listBuckets(buckets []*s3.Bucket) ([]*s3.Bucket, error) { + + input := &s3.ListBucketsInput{} + + resp, err := s.client.ListBuckets(input) + if err != nil { + log.WithField("error", err).Error("could not list s3 buckets") + return nil, err + } + + if buckets == nil { + buckets = []*s3.Bucket{} + } + + buckets = append(buckets, resp.Buckets...) + + return buckets, nil +} diff --git a/collector/aws/resources/s3_test.go b/collector/aws/resources/s3_test.go new file mode 100644 index 00000000..7ee9ec5c --- /dev/null +++ b/collector/aws/resources/s3_test.go @@ -0,0 +1,145 @@ +package resources + +import ( + "errors" + awsTestutils "finala/collector/aws/testutils" + "finala/collector/config" + "finala/collector/testutils" + collectorTestutils "finala/collector/testutils" + awsClient "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "reflect" + "testing" + "time" +) + +var defaultS3Mock = s3.ListBucketsOutput{ + Buckets: []*s3.Bucket{ + { + Name: awsClient.String("test-bucket"), + CreationDate: testutils.TimePointer(time.Now()), + }, + }, +} + +type MockAWSS3Client struct { + responseListBuckets s3.ListBucketsOutput + err error +} + +func (r *MockAWSS3Client) ListBuckets(input *s3.ListBucketsInput) (*s3.ListBucketsOutput, error) { + return &r.responseListBuckets, r.err +} + +func TestS3ListBuckets(t *testing.T) { + + collector := collectorTestutils.NewMockCollector() + detector := awsTestutils.AWSManager(collector, nil, nil, "us-east-1") + + t.Run("valid", func(t *testing.T) { + + mockClient := MockAWSS3Client{ + responseListBuckets: defaultS3Mock, + } + + s3Interface, err := NewS3Manager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected s3 manager error happend, got %v expected %v", err, nil) + } + + s3Manager, ok := s3Interface.(*S3Manager) + if !ok { + t.Fatalf("unexpected s3 struct, got %s expected %s", reflect.TypeOf(s3Interface), "*S3Manager") + } + + result, _ := s3Manager.listBuckets(nil) + + if len(result) != len(defaultS3Mock.Buckets) { + t.Fatalf("unexpected S3 bucket count, got %d expected %d", len(result), len(defaultS3Mock.Buckets)) + } + + }) + + t.Run("error", func(t *testing.T) { + mockClient := MockAWSS3Client{ + responseListBuckets: defaultS3Mock, + err: errors.New("error"), + } + + s3, err := NewS3Manager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected s3 manager error happend, got %v expected %v", err, nil) + } + + s3Manager, ok := s3.(*S3Manager) + if !ok { + t.Fatalf("unexpected s3 struct, got %s expected %s", reflect.TypeOf(s3Manager), "*S3Manager") + } + + _, err = s3Manager.listBuckets(nil) + + if err == nil { + t.Fatalf("unexpected list buckets error, return empty") + } + }) + +} + +func TestDetectS3(t *testing.T) { + + collector := collectorTestutils.NewMockCollector() + mockCloudwatch := awsTestutils.NewMockCloudwatch(nil) + detector := awsTestutils.AWSManager(collector, mockCloudwatch, nil, "us-east-1") + + var defaultMetricConfig = []config.MetricConfig{ + { + Description: "test description write capacity", + Data: []config.MetricDataConfiguration{ + { + Name: "TestMetric", + Statistic: "Sum", + }, + }, + Constraint: config.MetricConstraintConfig{ + Operator: "==", + Value: 5, + }, + Period: 1, + StartTime: 1, + }, + } + + mockClient := MockAWSS3Client{ + responseListBuckets: defaultS3Mock, + } + + s3, err := NewS3Manager(detector, &mockClient) + if err != nil { + t.Fatalf("unexpected s3 manager error happend, got %v expected %v", err, nil) + } + + s3Manager, ok := s3.(*S3Manager) + if !ok { + t.Fatalf("unexpected s3 struct, got %s expected %s", reflect.TypeOf(s3Manager), "*S3Manager") + } + + response, _ := s3Manager.Detect(defaultMetricConfig) + + s3Response, ok := response.([]DetectedS3) + if !ok { + t.Fatalf("unexpected s3 struct, got %s expected %s", reflect.TypeOf(response), "[]DetectedS3") + } + + if len(s3Response) != 1 { + t.Fatalf("unexpected s3 detected, got %d expected %d", len(s3Response), 1) + } + + if len(collector.Events) != 1 { + t.Fatalf("unexpected collector s3 resources, got %d expected %d", len(collector.Events), 1) + } + + if len(collector.EventsCollectionStatus) != 2 { + t.Fatalf("unexpected resource status events count, got %d expected %d", len(collector.EventsCollectionStatus), 2) + } + +} diff --git a/collector/aws/testutils/detector.go b/collector/aws/testutils/detector.go index 4bab9eca..473e9385 100644 --- a/collector/aws/testutils/detector.go +++ b/collector/aws/testutils/detector.go @@ -17,11 +17,17 @@ type MockAWSManager struct { pricing *pricing.PricingManager session *session.Session accountIdentity *sts.GetCallerIdentityOutput + accountName string region string global map[string]struct{} } -func AWSManager(collector collector.CollectorDescriber, cloudWatchClient *cloudwatch.CloudwatchManager, priceClient *pricing.PricingManager, region string) *MockAWSManager { +func AWSManager( + collector collector.CollectorDescriber, + cloudWatchClient *cloudwatch.CloudwatchManager, + priceClient *pricing.PricingManager, + region string, +) *MockAWSManager { accountID := "1234" accountIdentity := &sts.GetCallerIdentityOutput{ @@ -34,6 +40,7 @@ func AWSManager(collector collector.CollectorDescriber, cloudWatchClient *cloudw pricing: priceClient, accountIdentity: accountIdentity, region: region, + accountName: "test", global: make(map[string]struct{}), } } @@ -58,6 +65,10 @@ func (dm *MockAWSManager) GetRegion() string { return dm.region } +func (dm *MockAWSManager) GetAccountName() string { + return dm.accountName +} + func (dm *MockAWSManager) GetSession() (*session.Session, *aws.Config) { return dm.session, &aws.Config{} } diff --git a/collector/collector.go b/collector/collector.go index b53b48c5..f3631f42 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -25,8 +25,8 @@ const ( // CollectorDescriber describe the collector functions type CollectorDescriber interface { AddResource(data EventCollector) - CollectStart(resourceName ResourceIdentifier) - CollectFinish(resourceName ResourceIdentifier) + CollectStart(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) + CollectFinish(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) CollectError(resourceName ResourceIdentifier, err error) GetCollectorEvent() []EventCollector } @@ -98,21 +98,23 @@ func (cm *CollectorManager) AddResource(data EventCollector) { } // CollectStart add `fetch` event to collector by given resource name -func (cm *CollectorManager) CollectStart(resourceName ResourceIdentifier) { +func (cm *CollectorManager) CollectStart(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) { cm.updateServiceStatus(EventCollector{ ResourceName: resourceName, Data: EventStatusData{ - Status: EventFetch, + Status: EventFetch, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) } // CollectFinish add `finish` event to collector by given resource name -func (cm *CollectorManager) CollectFinish(resourceName ResourceIdentifier) { +func (cm *CollectorManager) CollectFinish(resourceName ResourceIdentifier, accountSpecifiedFields AccountSpecifiedFields) { cm.updateServiceStatus(EventCollector{ ResourceName: resourceName, Data: EventStatusData{ - Status: EventFinish, + Status: EventFinish, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) } diff --git a/collector/collector_test.go b/collector/collector_test.go index 55224438..88931e85 100644 --- a/collector/collector_test.go +++ b/collector/collector_test.go @@ -78,7 +78,10 @@ func TestAddEvent(t *testing.T) { time.Sleep(time.Second) - coll.CollectStart(collector.ResourceIdentifier("test")) + coll.CollectStart(collector.ResourceIdentifier("test"), collector.AccountSpecifiedFields{ + AccountName: "Test", + AccountID: "1234567890", + }) coll.AddResource(collector.EventCollector{ ResourceName: "test1", Data: "test data", @@ -122,7 +125,10 @@ func TestAddEventServerUnavailable(t *testing.T) { time.Sleep(time.Second) - coll.CollectStart(collector.ResourceIdentifier("test")) + coll.CollectStart(collector.ResourceIdentifier("test"), collector.AccountSpecifiedFields{ + AccountName: "Test", + AccountID: "1234567890", + }) coll.AddResource(collector.EventCollector{ ResourceName: "test1", diff --git a/collector/structs.go b/collector/structs.go index 6ecd26f4..9dc29506 100644 --- a/collector/structs.go +++ b/collector/structs.go @@ -23,8 +23,9 @@ const ( // EventStatusData descrive the struct of the resource statuses type EventStatusData struct { - Status EventStatus - ErrorMessage string + Status EventStatus + ErrorMessage string + AccountInformation string } // PriceDetectedFields describe the pricing field @@ -36,6 +37,12 @@ type PriceDetectedFields struct { Tag map[string]string } +// AccountSpecifiedFields describe account data of an resource +type AccountSpecifiedFields struct { + AccountID string + AccountName string +} + // EventCollector collector event data structure type EventCollector struct { EventType string diff --git a/collector/testutils/collector.go b/collector/testutils/collector.go index 0a41c9bb..81b7a9cc 100644 --- a/collector/testutils/collector.go +++ b/collector/testutils/collector.go @@ -23,20 +23,22 @@ func (mc *MockCollector) GetCollectorEvent() []collector.EventCollector { return events } -func (mc *MockCollector) CollectStart(resourceName collector.ResourceIdentifier) { +func (mc *MockCollector) CollectStart(resourceName collector.ResourceIdentifier, accountSpecifiedFields collector.AccountSpecifiedFields) { mc.updateServiceStatus(collector.EventCollector{ ResourceName: resourceName, Data: collector.EventStatusData{ - Status: collector.EventFetch, + Status: collector.EventFetch, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) } -func (mc *MockCollector) CollectFinish(resourceName collector.ResourceIdentifier) { +func (mc *MockCollector) CollectFinish(resourceName collector.ResourceIdentifier, accountSpecifiedFields collector.AccountSpecifiedFields) { mc.updateServiceStatus(collector.EventCollector{ ResourceName: resourceName, Data: collector.EventStatusData{ - Status: collector.EventFinish, + Status: collector.EventFinish, + AccountInformation: accountSpecifiedFields.AccountName + "_" + accountSpecifiedFields.AccountID, }, }) diff --git a/configuration/collector.yaml b/configuration/collector.yaml index 95557126..6f8e306c 100644 --- a/configuration/collector.yaml +++ b/configuration/collector.yaml @@ -210,4 +210,15 @@ providers: start_time: 168h # 24h * 7d constraint: operator: "==" - value: 0 + value: 0 + s3: + - description: Number of Requests + enable: true + metrics: + - name: AllRequests + statistic: Sum + period: 24h + start_time: 168h # 24h * 7d + constraint: + operator: "==" + value: 0 diff --git a/interpolation/interpolation.go b/interpolation/interpolation.go index f7268f17..8c936354 100644 --- a/interpolation/interpolation.go +++ b/interpolation/interpolation.go @@ -59,3 +59,12 @@ func ExtractExecutionName(executionId string) (string, error) { return executionArr[0], nil } + +// ExtractAccountInformation will return Account Name and Account ID +func ExtractAccountInformation(account string) (string, string, error) { + info := strings.Split(account, "_") + if len(info) != 2 { + return "", "", errors.New("unexpected account format") + } + return info[0], info[1], nil +} diff --git a/interpolation/interpolation_test.go b/interpolation/interpolation_test.go index 51e5d4db..221c64fd 100644 --- a/interpolation/interpolation_test.go +++ b/interpolation/interpolation_test.go @@ -71,3 +71,28 @@ func TestExtractExecutionName(t *testing.T) { t.Errorf("extractedExecutionName %s is not equal to expected timestamp %s", extractedExecutionName, index_prefix) } } + +func TestExtractAccountInformation(t *testing.T) { + const name, id = "Test", "1234567890" + //test for right input + accountInfo := fmt.Sprintf("%s_%s", name, id) + extractedName, extractedId, err := interpolation.ExtractAccountInformation(accountInfo) + if err != nil { + t.Fatalf("error occured while running ExtractAccountInformation e: %s\n", err) + } + + if extractedName != name { + t.Errorf("extractedName %s is not equal to expected name %s", extractedName, name) + } + + if extractedId != id { + t.Errorf("extractedId %s is not equal to expected id %s", extractedId, id) + } + + //test for wrong input + const wrong = "noUnderScore" + _, _, err = interpolation.ExtractAccountInformation(wrong) + if err == nil { + t.Errorf("ExtractAccountInformation returns no error for input without underscore: %s", wrong) + } +} diff --git a/ui/src/components/Dashboard/AccountsList.js b/ui/src/components/Dashboard/AccountsList.js new file mode 100644 index 00000000..025b0170 --- /dev/null +++ b/ui/src/components/Dashboard/AccountsList.js @@ -0,0 +1,91 @@ +import React, { Fragment } from "react"; +import { connect } from "react-redux"; +import PropTypes from "prop-types"; +import colors from "./colors.json"; +import { makeStyles } from "@material-ui/core/styles"; +import { Box, Chip } from "@material-ui/core"; +import { setHistory } from "../../utils/History"; + +const useStyles = makeStyles(() => ({ + title: { + fontFamily: "MuseoModerno", + }, + resource_chips: { + fontWeight: "bold", + fontFamily: "Arial !important", + margin: "5px", + borderRadius: "1px", + backgroundColor: "#ffffff", + borderLeft: "5px solid #ffffff", + fontSize: "14px", + }, +})); + +/** + * @param {array} accounts Accounts List + * @param {array} filters Filters List + * @param {func} addFilter Add filter to filters list + */ +const AccountsList = ({ accounts, filters, addFilter }) => { + const classes = useStyles(); + + const accountsList = Object.values(accounts).map((account) => { + account.title = `${account.Name}(${account.ID})`; + return account; + }); + + /** + * + * @param {object} account add selected account + */ + const setSelectedAccount = (account) => { + const filter = { + title: `Account:${account.title}`, + id: `account:${account.ID}`, + type: "account", + }; + + addFilter(filter); + + setHistory({ + filters: filters, + }); + }; + + return ( + + {accountsList.length > 0 && ( + +

Accounts:

+ {accountsList.map((account, i) => ( + setSelectedAccount(account)} + ma={2} + label={account.title} + key={i} + /> + ))} + + )} + + ); +}; + +AccountsList.defaultProps = {}; +AccountsList.propTypes = { + accounts: PropTypes.object, + filters: PropTypes.array, + addFilter: PropTypes.func, +}; + +const mapStateToProps = (state) => ({ + accounts: state.accounts.accounts, + filters: state.filters.filters, +}); +const mapDispatchToProps = (dispatch) => ({ + addFilter: (data) => dispatch({ type: "ADD_FILTER", data }), +}); + +export default connect(mapStateToProps, mapDispatchToProps)(AccountsList); diff --git a/ui/src/components/Dashboard/FilterBar.js b/ui/src/components/Dashboard/FilterBar.js index ca2f9137..d029cb76 100644 --- a/ui/src/components/Dashboard/FilterBar.js +++ b/ui/src/components/Dashboard/FilterBar.js @@ -137,6 +137,15 @@ const FilterBar = ({ type: "resource", }); resource = filterValue; + } else if (filterValue && filterKey === "account") { + const accounts = filterValue.split(","); + accounts.forEach((account) => { + filters.push({ + title: `Account:${account}`, + id: `account:${account}`, + type: "account", + }); + }); } else if (filterValue) { const filterValues = filterValue.split(","); diff --git a/ui/src/components/Dashboard/Index.js b/ui/src/components/Dashboard/Index.js index a137f8e9..5d48e9b8 100644 --- a/ui/src/components/Dashboard/Index.js +++ b/ui/src/components/Dashboard/Index.js @@ -4,10 +4,11 @@ import { makeStyles } from "@material-ui/core/styles"; import { setHistory } from "../../utils/History"; import PropTypes from "prop-types"; +import AccountsList from "./AccountsList"; import FilterBar from "./FilterBar"; import StatisticsBar from "./StatisticsBar"; import ResourceScanning from "./ResourceScanning"; -import ResourcesChart from "./ResourcesChart"; +import ResourcesCharts from "./ResourcesCharts"; import ResourcesList from "./ResourcesList"; import ResourceTable from "./ResourceTable"; import ExecutionIndex from "../Executions/Index"; @@ -71,8 +72,9 @@ const DashboardIndex = ({ + - {currentResource ? : } + {currentResource ? : } ); }; diff --git a/ui/src/components/Dashboard/ResourcesChart.js b/ui/src/components/Dashboard/ResourcesChart.js index 42140213..92a76515 100644 --- a/ui/src/components/Dashboard/ResourcesChart.js +++ b/ui/src/components/Dashboard/ResourcesChart.js @@ -17,6 +17,9 @@ import { import ReportProblemIcon from "@material-ui/icons/ReportProblem"; const useStyles = makeStyles(() => ({ + title: { + fontFamily: "MuseoModerno", + }, noDataTitle: { textAlign: "center", fontWeight: "bold", @@ -38,19 +41,35 @@ const useStyles = makeStyles(() => ({ * @param {bool} isResourceListLoading isLoading state for resources * @param {func} addFilter Add filter to filters list * @param {func} setResource Update Selected Resource} + * @param {string} account Account ID for account specific summary + * @param {object} accounts Accounts of current execution */ const ResourcesChart = ({ resources, filters, + setFilters, isResourceListLoading, addFilter, setResource, + account, + accounts, }) => { const classes = useStyles(); const colorList = colors.map((color) => color.hex); - const sortedResources = Object.values(resources) - .filter((row) => row.TotalSpent > 0) - .sort((a, b) => (a.TotalSpent >= b.TotalSpent ? -1 : 1)); + let sortedResources; + if (account) { + sortedResources = Object.values(resources) + .filter( + (row) => row.SpentAccounts[account] && row.SpentAccounts[account] > 0 + ) + .sort((a, b) => + a.SpentAccounts[account] >= b.SpentAccounts[account] ? -1 : 1 + ); + } else { + sortedResources = Object.values(resources) + .filter((row) => row.TotalSpent > 0) + .sort((a, b) => (a.TotalSpent >= b.TotalSpent ? -1 : 1)); + } const chartOptions = { options: { @@ -63,6 +82,18 @@ const ResourcesChart = ({ const res = sortedResources; const selectedResource = res[dataPointIndex]; setSelectedResource(selectedResource); + if (account) { + const nfilters = filters.filter( + (filter) => filter.type !== "account" + ); + setFilters(nfilters); + const filter = { + title: `Account:${account}`, + id: `account:${account}`, + type: "account", + }; + addFilter(filter); + } }, }, }, @@ -148,28 +179,43 @@ const ResourcesChart = ({ */ sortedResources.forEach((resource) => { const title = titleDirective(resource.ResourceName); - const amount = MoneyDirective(resource.TotalSpent); + const amount = MoneyDirective( + account ? resource.SpentAccounts[account] : resource.TotalSpent + ); resource.title = `${title} (${amount})`; resource.display_title = `${title}`; chartOptions.options.xaxis.categories.push(resource.title); - chartOptions.series[0].data.push(resource.TotalSpent); + chartOptions.series[0].data.push( + account ? resource.SpentAccounts[account] : resource.TotalSpent + ); return resource; }); + if (account && !sortedResources.length && !isResourceListLoading) { + return ; + } + return ( {!isResourceListLoading && sortedResources.length > 0 && ( - + +

+ {account + ? `${accounts[account].Name} (${accounts[account].ID}):` + : "Summary:"} +

+ + )} {isResourceListLoading && ( @@ -191,18 +237,23 @@ ResourcesChart.defaultProps = {}; ResourcesChart.propTypes = { resources: PropTypes.object, filters: PropTypes.array, + setFilters: PropTypes.func, isResourceListLoading: PropTypes.bool, addFilter: PropTypes.func, setResource: PropTypes.func, + account: PropTypes.string, + accounts: PropTypes.object, }; const mapStateToProps = (state) => ({ resources: state.resources.resources, isResourceListLoading: state.resources.isResourceListLoading, filters: state.filters.filters, + accounts: state.accounts.accounts, }); const mapDispatchToProps = (dispatch) => ({ + setFilters: (data) => dispatch({ type: "SET_FILTERS", data }), addFilter: (data) => dispatch({ type: "ADD_FILTER", data }), setResource: (data) => dispatch({ type: "SET_RESOURCE", data }), }); diff --git a/ui/src/components/Dashboard/ResourcesCharts.js b/ui/src/components/Dashboard/ResourcesCharts.js new file mode 100644 index 00000000..1ba379d6 --- /dev/null +++ b/ui/src/components/Dashboard/ResourcesCharts.js @@ -0,0 +1,36 @@ +import React from "react"; +import { connect } from "react-redux"; +import PropTypes from "prop-types"; +import { Fragment } from "react"; +import ResourcesChart from "./ResourcesChart"; + +/** + * @param {accounts} object Accounts of current execution + * @param {filters} array Filters list + */ +const ResourcesCharts = ({ accounts, filters }) => { + let selectedAccountIds = filters + .filter((filter) => filter.type === "account") + .map((filter) => filter.id.split(":")[1]); + if (selectedAccountIds.length === 0) { + selectedAccountIds = Object.keys(accounts); + } + let resourcesCharts = selectedAccountIds.map((accountID) => ( + + )); + resourcesCharts = [, ...resourcesCharts]; + return {resourcesCharts}; +}; + +ResourcesCharts.defaultProps = {}; +ResourcesCharts.propTypes = { + filters: PropTypes.array, + accounts: PropTypes.object, +}; + +const mapStateToProps = (state) => ({ + filters: state.filters.filters, + accounts: state.accounts.accounts, +}); + +export default connect(mapStateToProps)(ResourcesCharts); diff --git a/ui/src/components/DataFactory.js b/ui/src/components/DataFactory.js index dd050fd1..1e90ccf0 100644 --- a/ui/src/components/DataFactory.js +++ b/ui/src/components/DataFactory.js @@ -5,6 +5,7 @@ import { ResourcesService } from "services/resources.service"; import { SettingsService } from "services/settings.service"; import { titleDirective } from "utils/Title"; import { getHistory, setHistory } from "../utils/History"; +import { AccsService } from "../services/accs.service"; let fetchTimeoutRequest = false; let fetchTableTimeoutRequest = false; @@ -19,6 +20,7 @@ let lastFiltersSearched = "[]"; * @param {func} setCurrentExecution Update Current Execution * * @param {string} currentResource Current selected resource + * @param {func} setAccounts Update Accounts List * @param {func} setResources Update Resources List * @param {func} setCurrentResourceData Update current resource data * @param {func} setIsResourceListLoading update isLoading state for resources @@ -38,6 +40,7 @@ const DataFacotry = ({ setCurrentExecution, currentResource, + setAccounts, setResources, setCurrentResourceData, setIsResourceListLoading, @@ -114,6 +117,7 @@ const DataFacotry = ({ clearTimeout(fetchTimeoutRequest); setIsResourceListLoading(true); await getResources(currentExecution, filters); + await getAccounts(currentExecution); setIsResourceListLoading(false); if (currentResource) { @@ -140,6 +144,24 @@ const DataFacotry = ({ setIsResourceTableLoading(false); }; + /** + * Will fetch account list from server + * @param {string} currentExecution current Selected Execution + */ + const getAccounts = async (currentExecution) => { + const AccountsArray = await AccsService.list(currentExecution).catch( + () => false + ); + + const accounts = {}; + AccountsArray.forEach((value) => { + accounts[value.ID] = value; + }); + + setAccounts(accounts); + return true; + }; + /** * Will fetch resource list from server * @param {string} currentExecution Current Selected Execution @@ -258,11 +280,13 @@ DataFacotry.propTypes = { setIsResourceListLoading: PropTypes.func, setIsResourceTableLoading: PropTypes.func, setIsScanning: PropTypes.func, + setAccounts: PropTypes.func, setResources: PropTypes.func, setCurrentResourceData: PropTypes.func, setCurrentExecution: PropTypes.func, currentResource: PropTypes.string, + accounts: PropTypes.object, resources: PropTypes.object, filters: PropTypes.array, currentExecution: PropTypes.string, @@ -273,6 +297,7 @@ DataFacotry.propTypes = { }; const mapStateToProps = (state) => ({ + accounts: state.accounts.accounts, resources: state.resources.resources, currentResource: state.resources.currentResource, currentExecution: state.executions.current, @@ -289,6 +314,7 @@ const mapDispatchToProps = (dispatch) => ({ setIsResourceTableLoading: (isLoading) => dispatch({ type: "IS_RESOURCE_TABLE_LOADING", isLoading }), setIsScanning: (isScanning) => dispatch({ type: "IS_SCANNING", isScanning }), + setAccounts: (data) => dispatch({ type: "ACCOUNT_LIST", data }), setResources: (data) => dispatch({ type: "RESOURCE_LIST", data }), setCurrentExecution: (id) => dispatch({ type: "EXECUTION_SELECTED", id }), setCurrentResourceData: (data) => diff --git a/ui/src/reducers/accounts.reducer.js b/ui/src/reducers/accounts.reducer.js new file mode 100644 index 00000000..b88b24ff --- /dev/null +++ b/ui/src/reducers/accounts.reducer.js @@ -0,0 +1,18 @@ +const initialState = { + accounts: {}, +}; + +/** + * @param {object} state module state + * @param {object} action to apply on state + * @returns {object} new copy of state + */ +export function accounts(state = initialState, action) { + switch (action.type) { + case "ACCOUNT_LIST": + state.accounts = action.data; + return { ...state }; + default: + return state; + } +} diff --git a/ui/src/reducers/index.js b/ui/src/reducers/index.js index f894e2dd..ad94d29c 100755 --- a/ui/src/reducers/index.js +++ b/ui/src/reducers/index.js @@ -1,11 +1,13 @@ import { combineReducers } from "redux"; import { connectRouter } from "connected-react-router"; +import { accounts } from "../reducers/accounts.reducer"; import { resources } from "../reducers/resources.reducer"; import { executions } from "../reducers/executions.reducer"; import { filters } from "../reducers/filters.reducer"; const rootReducer = (history) => combineReducers({ + accounts, resources, executions, filters, diff --git a/ui/src/services/accs.service.js b/ui/src/services/accs.service.js new file mode 100644 index 00000000..89666cd3 --- /dev/null +++ b/ui/src/services/accs.service.js @@ -0,0 +1,18 @@ +import { http } from "./request.service"; + +export const AccsService = { + list, +}; + +/** + * + * @param {string} executionId execution to query + */ +function list(executionId) { + return http + .send(`api/v1/accounts/${executionId}`, `get`) + .then(this.handleResponse) + .then((response) => { + return response; + }); +} diff --git a/ui/src/services/resources.service.js b/ui/src/services/resources.service.js index 878aa72b..8ab79481 100644 --- a/ui/src/services/resources.service.js +++ b/ui/src/services/resources.service.js @@ -14,12 +14,17 @@ export const ResourcesService = { const getTransformedFilters = (filters) => { const params = {}; filters.forEach((filter) => { - if (filter.id.substr(0, 8) === "resource") { + if (filter.type === "resource") { return; } const [key, value] = filter.id.split(":"); + let paramKey; + if (value && filter.type === "account") { + paramKey = `filter_Data.AccountID`; + } else { + paramKey = `filter_Data.Tag.${key}`; + } if (value) { - const paramKey = `filter_Data.Tag.${key}`; if (params[paramKey]) { params[paramKey] += `,${value}`; } else {