diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 61e88d7b76d0..43d948c18f59 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -231,6 +231,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Add beta ingest_pipeline metricset to Elasticsearch module for ingest pipeline monitoring {pull}34012[34012] - Handle duplicated TYPE line for prometheus metrics {issue}18813[18813] {pull}33865[33865] - Support collecting metrics from both the monitoring account and linked accounts from AWS CloudWatch. {pull}35540[35540] +- Add new parameter `include_linked_accounts` to enable/disable metrics collection from multiple linked AWS Accounts {pull}35648[35648] *Packetbeat* diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 9b8c4e5ac686..d67dfb41eee2 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -73,7 +73,14 @@ services do not include a region. In `aws` module, `endpoint` config is to set the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, `sc2s.sgov.gov`. -If endpoint is specified, `regions` config becomes required. For example: +If endpoint is specified, `regions` config becomes required. + +* *include_linked_accounts* + +The `include_linked_accounts` parameter is used to enable the inclusion of metrics from different accounts linked to a main monitoring account. By setting this parameter to true, users can gather metrics from multiple AWS accounts that are linked through the https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Cross-Account-Cross-Region.html[Cross-Account Cross-Region feature in Amazon CloudWatch]. +By default, the `include_linked_accounts` parameter is set to false, meaning that only metrics from the main monitoring account are collected. When set to true, the parameter allows the CloudWatch service to access and retrieve metrics from the linked accounts, providing a comprehensive view of metrics across multiple accounts. + +*_Note_:* Enabling the include_linked_accounts parameter requires appropriate permissions and authentication credentials for the linked accounts. Users should ensure that the necessary IAM roles and policies are properly set up to establish the necessary trust and access between accounts. * *tags_filter* @@ -413,6 +420,12 @@ metricbeat.modules: - transitgateway - usage - vpn +- module: aws + period: 1m + latency: 5m + include_linked_accounts: false + metricsets: + - s3_request ---- [float] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 3a275347ff00..dfd6edc07260 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -237,6 +237,12 @@ metricbeat.modules: - transitgateway - usage - vpn +- module: aws + period: 1m + latency: 5m + include_linked_accounts: false + metricsets: + - s3_request #----------------------------- AWS Fargate Module ----------------------------- - module: awsfargate diff --git a/x-pack/metricbeat/module/aws/_meta/config.reference.yml b/x-pack/metricbeat/module/aws/_meta/config.reference.yml index de3f9f1ed347..12866939b98b 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.reference.yml @@ -51,3 +51,9 @@ - transitgateway - usage - vpn +- module: aws + period: 1m + latency: 5m + include_linked_accounts: false + metricsets: + - s3_request diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 68293513b027..6adf3af2fcd5 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -49,5 +49,6 @@ - module: aws period: 1m latency: 5m + include_linked_accounts: false metricsets: - s3_request diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index c19e80d5db3f..c55570b3d356 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -61,7 +61,14 @@ services do not include a region. In `aws` module, `endpoint` config is to set the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, `sc2s.sgov.gov`. -If endpoint is specified, `regions` config becomes required. For example: +If endpoint is specified, `regions` config becomes required. + +* *include_linked_accounts* + +The `include_linked_accounts` parameter is used to enable the inclusion of metrics from different accounts linked to a main monitoring account. By setting this parameter to true, users can gather metrics from multiple AWS accounts that are linked through the https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Cross-Account-Cross-Region.html[Cross-Account Cross-Region feature in Amazon CloudWatch]. +By default, the `include_linked_accounts` parameter is set to false, meaning that only metrics from the main monitoring account are collected. When set to true, the parameter allows the CloudWatch service to access and retrieve metrics from the linked accounts, providing a comprehensive view of metrics across multiple accounts. + +*_Note_:* Enabling the include_linked_accounts parameter requires appropriate permissions and authentication credentials for the linked accounts. Users should ensure that the necessary IAM roles and policies are properly set up to establish the necessary trust and access between accounts. * *tags_filter* diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 70934dd0f407..76f0b9d83d84 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -29,26 +29,28 @@ type describeRegionsClient interface { // Config defines all required and optional parameters for aws metricsets type Config struct { - Period time.Duration `config:"period" validate:"nonzero,required"` - DataGranularity time.Duration `config:"data_granularity"` - Regions []string `config:"regions"` - Latency time.Duration `config:"latency"` - AWSConfig awscommon.ConfigAWS `config:",inline"` - TagsFilter []Tag `config:"tags_filter"` + Period time.Duration `config:"period" validate:"nonzero,required"` + DataGranularity time.Duration `config:"data_granularity"` + Regions []string `config:"regions"` + Latency time.Duration `config:"latency"` + AWSConfig awscommon.ConfigAWS `config:",inline"` + TagsFilter []Tag `config:"tags_filter"` + IncludeLinkedAccounts *bool `config:"include_linked_accounts"` } // MetricSet is the base metricset for all aws metricsets type MetricSet struct { mb.BaseMetricSet - RegionsList []string - Endpoint string - Period time.Duration - DataGranularity time.Duration - Latency time.Duration - AwsConfig *awssdk.Config - AccountName string - AccountID string - TagsFilter []Tag + RegionsList []string + Endpoint string + Period time.Duration + DataGranularity time.Duration + Latency time.Duration + AwsConfig *awssdk.Config + MonitoringAccountName string + MonitoringAccountID string + TagsFilter []Tag + IncludeLinkedAccounts bool } // Tag holds a configuration specific for ec2 and cloudwatch metricset. @@ -60,6 +62,39 @@ type Tag struct { // ModuleName is the name of this module. const ModuleName = "aws" +// IncludeLinkedAccountsDefault defines if we should include metrics from linked AWS accounts or not. Default is false. +// More information about cross-account Cloudwatch monitoring can be found at +// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Cross-Account-Cross-Region.html +const IncludeLinkedAccountsDefault = false + +type LabelConstants struct { + AccountIdIdx int + AccountLabelIdx int + MetricNameIdx int + NamespaceIdx int + StatisticIdx int + IdentifierNameIdx int + IdentifierValueIdx int + LabelLengthTotal int + LabelSeparator string + AccountLabel string + BillingDimensionStartIdx int +} + +var LabelConst = LabelConstants{ + AccountIdIdx: 0, + AccountLabelIdx: 1, + MetricNameIdx: 2, + NamespaceIdx: 3, + StatisticIdx: 4, + IdentifierNameIdx: 5, + IdentifierValueIdx: 6, + LabelLengthTotal: 7, + LabelSeparator: "|", + AccountLabel: "${PROP('AccountLabel')}", + BillingDimensionStartIdx: 3, +} + func init() { if err := mb.Registry.AddModule(ModuleName, newModule); err != nil { panic(err) @@ -112,10 +147,16 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { Endpoint: config.AWSConfig.Endpoint, } + metricSet.IncludeLinkedAccounts = IncludeLinkedAccountsDefault + if config.IncludeLinkedAccounts != nil { + metricSet.IncludeLinkedAccounts = *config.IncludeLinkedAccounts + } + base.Logger().Debug("Metricset level config for period: ", metricSet.Period) base.Logger().Debug("Metricset level config for data granularity: ", metricSet.DataGranularity) base.Logger().Debug("Metricset level config for tags filter: ", metricSet.TagsFilter) base.Logger().Warn("extra charges on AWS API requests will be generated by this metricset") + base.Logger().Debug("Metricset level config for including linked accounts: ", metricSet.IncludeLinkedAccounts) // If regions in config is not empty, then overwrite the awsConfig.Region if len(config.Regions) > 0 { @@ -134,8 +175,8 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { if err != nil { base.Logger().Warn("failed to get caller identity, please check permission setting: ", err) } else { - metricSet.AccountID = *outputIdentity.Account - base.Logger().Debug("AWS Credentials belong to account ID: ", metricSet.AccountID) + metricSet.MonitoringAccountID = *outputIdentity.Account + base.Logger().Debug("AWS Credentials belong to monitoring account ID: ", metricSet.MonitoringAccountID) } // Get account name/alias svcIam := iam.NewFromConfig(awsConfig, func(o *iam.Options) { @@ -144,7 +185,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { } }) - metricSet.AccountName = getAccountName(svcIam, base, metricSet) + metricSet.MonitoringAccountName = getAccountName(svcIam, base, metricSet) // Construct MetricSet with a full regions list if config.Regions == nil { @@ -188,7 +229,7 @@ func getAccountName(svc *iam.Client, base mb.BaseMetricSet, metricSet MetricSet) defer cancel() output, err := svc.ListAccountAliases(ctx, &iam.ListAccountAliasesInput{}) - accountName := metricSet.AccountID + accountName := metricSet.MonitoringAccountID if err != nil { base.Logger().Warn("failed to list account aliases, please check permission setting: ", err) return accountName @@ -196,15 +237,15 @@ func getAccountName(svc *iam.Client, base mb.BaseMetricSet, metricSet MetricSet) // When there is no account alias, account ID will be used as cloud.account.name if len(output.AccountAliases) == 0 { - accountName = metricSet.AccountID - base.Logger().Debug("AWS Credentials belong to account ID: ", metricSet.AccountID) + accountName = metricSet.MonitoringAccountID + base.Logger().Debug("AWS Credentials belong to account ID: ", metricSet.MonitoringAccountID) return accountName } // There can be more than one aliases for each account, for now we are only // collecting the first one. accountName = output.AccountAliases[0] - base.Logger().Debug("AWS Credentials belong to account name: ", metricSet.AccountName) + base.Logger().Debug("AWS Credentials belong to account name: ", metricSet.MonitoringAccountName) return accountName } diff --git a/x-pack/metricbeat/module/aws/billing/billing.go b/x-pack/metricbeat/module/aws/billing/billing.go index c5d3d368dc63..75b500839ef5 100644 --- a/x-pack/metricbeat/module/aws/billing/billing.go +++ b/x-pack/metricbeat/module/aws/billing/billing.go @@ -28,9 +28,8 @@ import ( ) var ( - metricsetName = "billing" - regionName = "us-east-1" - labelSeparator = "|" + metricsetName = "billing" + regionName = "us-east-1" // This list is from https://github.com/aws/aws-sdk-go-v2/blob/master/service/costexplorer/api_enums.go#L60-L90 supportedDimensionKeys = []string{ @@ -42,11 +41,7 @@ var ( "RESERVATION_ID", } - dateLayout = "2006-01-02" - accountIdIdx = 0 - accountLabelIdx = 1 - metricDataValueIdx = 2 - dimensionStartIdx = 3 + dateLayout = "2006-01-02" ) // init registers the MetricSet with the central registry as soon as the program @@ -175,7 +170,7 @@ func (m *MetricSet) getCloudWatchBillingMetrics( endTime time.Time) []mb.Event { var events []mb.Event namespace := "AWS/Billing" - listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, svcCloudwatch) + listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, m.IncludeLinkedAccounts, m.MonitoringAccountID, svcCloudwatch) if err != nil { m.Logger().Error(err.Error()) return nil @@ -198,17 +193,17 @@ func (m *MetricSet) getCloudWatchBillingMetrics( continue } for valI, metricDataResultValue := range output.Values { - labels := strings.Split(*output.Label, labelSeparator) + labels := strings.Split(*output.Label, aws.LabelConst.LabelSeparator) event := mb.Event{} - if labels[accountIdIdx] != "" { - event = aws.InitEvent("", labels[accountLabelIdx], labels[accountIdIdx], output.Timestamps[valI]) + if labels[aws.LabelConst.AccountIdIdx] != "" { + event = aws.InitEvent("", labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI]) } else { - event = aws.InitEvent("", m.AccountName, m.AccountID, output.Timestamps[valI]) + event = aws.InitEvent("", m.MonitoringAccountName, m.MonitoringAccountID, output.Timestamps[valI]) } - _, _ = event.MetricSetFields.Put(labels[metricDataValueIdx], metricDataResultValue) + _, _ = event.MetricSetFields.Put(labels[aws.LabelConst.MetricNameIdx], metricDataResultValue) - i := dimensionStartIdx + i := aws.LabelConst.BillingDimensionStartIdx for i < len(labels)-1 { _, _ = event.MetricSetFields.Put(labels[i], labels[i+1]) i += 2 @@ -318,7 +313,7 @@ func (m *MetricSet) getCostGroupBy(svcCostExplorer *costexplorer.Client, groupBy } func (m *MetricSet) addCostMetrics(metrics map[string]costexplorertypes.MetricValue, groupDefinition costexplorertypes.GroupDefinition, startDate string, endDate string) mb.Event { - event := aws.InitEvent("", m.AccountName, m.AccountID, time.Now()) + event := aws.InitEvent("", m.MonitoringAccountName, m.MonitoringAccountID, time.Now()) // add group definition _, _ = event.MetricSetFields.Put("group_definition", mapstr.M{ @@ -367,9 +362,9 @@ func createMetricDataQuery(metric aws.MetricWithID, index int, dataGranularity t metricDims := metric.Metric.Dimensions metricName := *metric.Metric.MetricName - label := metric.AccountID + labelSeparator + "${PROP('AccountLabel')}" + labelSeparator + metricName + labelSeparator + label := strings.Join([]string{metric.AccountID, aws.LabelConst.AccountLabel, metricName}, aws.LabelConst.LabelSeparator) for _, dim := range metricDims { - label += *dim.Name + labelSeparator + *dim.Value + labelSeparator + label += aws.LabelConst.LabelSeparator + *dim.Name + aws.LabelConst.LabelSeparator + *dim.Value } metricDataQuery := types.MetricDataQuery{ diff --git a/x-pack/metricbeat/module/aws/billing/billing_integration_test.go b/x-pack/metricbeat/module/aws/billing/billing_integration_test.go index d9056ff38164..b782fd826f73 100644 --- a/x-pack/metricbeat/module/aws/billing/billing_integration_test.go +++ b/x-pack/metricbeat/module/aws/billing/billing_integration_test.go @@ -25,6 +25,7 @@ func TestFetch(t *testing.T) { metricSet := mbtest.NewReportingMetricSetV2Error(t, config) events, errs := mbtest.ReportingFetchV2Error(metricSet) + fmt.Println("events = ", events) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index fd4a35d07fc4..b277dd7b2ad1 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -25,18 +25,9 @@ import ( var ( metricsetName = "cloudwatch" - accountIdIdx = 0 - accountLabelIdx = 1 - metricNameIdx = 2 - namespaceIdx = 3 - statisticIdx = 4 - identifierNameIdx = 5 - identifierValueIdx = 6 defaultStatistics = []string{"Average", "Maximum", "Minimum", "Sum", "SampleCount"} - labelSeparator = "|" dimensionSeparator = "," dimensionValueWildcard = "*" - labelLengthTotal = 7 ) // init registers the MetricSet with the central registry as soon as the program @@ -51,7 +42,7 @@ func init() { // MetricSet holds any configuration or state information. It must implement // the mb.MetricSet interface. And this is best achieved by embedding -// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// mb.BaseMetricSet because it implements all the required mb.MetricSet // interface methods except for Fetch. type MetricSet struct { *aws.MetricSet @@ -186,7 +177,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } // retrieve all the details for all the metrics available in the current region - listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, svcCloudwatch) + listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.MonitoringAccountID, svcCloudwatch) if err != nil { m.Logger().Errorf("Error while retrieving the list of metrics for region %s: %w", regionName, err) } @@ -327,17 +318,17 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n !configDimensionValueContainsWildcard(config.Dimensions) { namespace := config.Namespace for i := range config.MetricName { - cwMetric := aws.MetricWithID{ - Metric: types.Metric{ - Namespace: &namespace, - MetricName: &config.MetricName[i], - Dimensions: cloudwatchDimensions, - }, + metric := types.Metric{ + Namespace: &namespace, + MetricName: &config.MetricName[i], + Dimensions: cloudwatchDimensions, } metricsWithStats := metricsWithStatistics{ - cloudwatchMetric: cwMetric, - statistic: config.Statistic, + cloudwatchMetric: aws.MetricWithID{ + Metric: metric, + }, + statistic: config.Statistic, } metricsWithStatsTotal = append(metricsWithStatsTotal, metricsWithStats) @@ -396,7 +387,7 @@ func createMetricDataQueries(listMetricsTotal []metricsWithStatistics, dataGranu func constructLabel(metric aws.MetricWithID, statistic string) string { // label = accountID + accountLabel + metricName + namespace + statistic + dimKeys + dimValues - label := strings.Join([]string{metric.AccountID, "${PROP('AccountLabel')}", *metric.Metric.MetricName, *metric.Metric.Namespace, statistic}, labelSeparator) + label := strings.Join([]string{metric.AccountID, aws.LabelConst.AccountLabel, *metric.Metric.MetricName, *metric.Metric.Namespace, statistic}, aws.LabelConst.LabelSeparator) dimNames := "" dimValues := "" for i, dim := range metric.Metric.Dimensions { @@ -409,8 +400,8 @@ func constructLabel(metric aws.MetricWithID, statistic string) string { } if dimNames != "" && dimValues != "" { - label += labelSeparator + dimNames - label += labelSeparator + dimValues + label += aws.LabelConst.LabelSeparator + dimNames + label += aws.LabelConst.LabelSeparator + dimValues } return label } @@ -432,12 +423,12 @@ func statisticLookup(stat string) (string, bool) { } func generateFieldName(namespace string, labels []string) string { - stat := labels[statisticIdx] + stat := labels[aws.LabelConst.StatisticIdx] // Check if statistic method is one of Sum, SampleCount, Minimum, Maximum, Average // With checkStatistics function, no need to check bool return value here statMethod, _ := statisticLookup(stat) // By default, replace dot "." using underscore "_" for metric names - return "aws." + stripNamespace(namespace) + ".metrics." + common.DeDot(labels[metricNameIdx]) + "." + statMethod + return "aws." + stripNamespace(namespace) + ".metrics." + common.DeDot(labels[aws.LabelConst.MetricNameIdx]) + "." + statMethod } // stripNamespace converts Cloudwatch namespace into the root field we will use for metrics @@ -448,15 +439,15 @@ func stripNamespace(namespace string) string { } func insertRootFields(event mb.Event, metricValue float64, labels []string) mb.Event { - namespace := labels[namespaceIdx] + namespace := labels[aws.LabelConst.NamespaceIdx] _, _ = event.RootFields.Put(generateFieldName(namespace, labels), metricValue) _, _ = event.RootFields.Put("aws.cloudwatch.namespace", namespace) - if len(labels) != labelLengthTotal { + if len(labels) != aws.LabelConst.LabelLengthTotal { return event } - dimNames := strings.Split(labels[identifierNameIdx], ",") - dimValues := strings.Split(labels[identifierValueIdx], ",") + dimNames := strings.Split(labels[aws.LabelConst.IdentifierNameIdx], ",") + dimValues := strings.Split(labels[aws.LabelConst.IdentifierValueIdx], ",") for i := 0; i < len(dimNames); i++ { _, _ = event.RootFields.Put("aws.dimensions."+dimNames[i], dimValues[i]) } @@ -487,25 +478,25 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient if len(metricDataResult.Values) == 0 { continue } - labels := strings.Split(*metricDataResult.Label, labelSeparator) + labels := strings.Split(*metricDataResult.Label, aws.LabelConst.LabelSeparator) for valI, metricDataResultValue := range metricDataResult.Values { - if len(labels) != labelLengthTotal { + if len(labels) != aws.LabelConst.LabelLengthTotal { // when there is no identifier value in label, use id+label+region+accountID+namespace+index instead - identifier := labels[accountIdIdx] + labels[accountLabelIdx] + regionName + m.AccountID + labels[namespaceIdx] + fmt.Sprint("-", valI) + identifier := labels[aws.LabelConst.AccountIdIdx] + labels[aws.LabelConst.AccountLabelIdx] + regionName + m.MonitoringAccountID + labels[aws.LabelConst.NamespaceIdx] + fmt.Sprint("-", valI) if _, ok := events[identifier]; !ok { - if labels[accountIdIdx] != "" { - events[identifier] = aws.InitEvent(regionName, labels[accountLabelIdx], labels[accountIdIdx], metricDataResult.Timestamps[valI]) + if labels[aws.LabelConst.AccountIdIdx] != "" { + events[identifier] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], metricDataResult.Timestamps[valI]) } else { - events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, metricDataResult.Timestamps[valI]) + events[identifier] = aws.InitEvent(regionName, m.MonitoringAccountName, m.MonitoringAccountID, metricDataResult.Timestamps[valI]) } } events[identifier] = insertRootFields(events[identifier], metricDataResultValue, labels) continue } - identifierValue := labels[identifierValueIdx] + fmt.Sprint("-", valI) + identifierValue := labels[aws.LabelConst.IdentifierValueIdx] + fmt.Sprint("-", valI) if _, ok := events[identifierValue]; !ok { - events[identifierValue] = aws.InitEvent(regionName, labels[accountLabelIdx], labels[accountIdIdx], metricDataResult.Timestamps[valI]) + events[identifierValue] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], metricDataResult.Timestamps[valI]) } events[identifierValue] = insertRootFields(events[identifierValue], metricDataResultValue, labels) } @@ -542,7 +533,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient continue } - labels := strings.Split(*output.Label, labelSeparator) + labels := strings.Split(*output.Label, aws.LabelConst.LabelSeparator) for valI, metricDataResultValue := range output.Values { if len(labels) != 7 { // if there is no tag in labels but there is a tagsFilter, then no event should be reported. @@ -551,19 +542,19 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient } // when there is no identifier value in label, use id+label+region+accountID+namespace+index instead - identifier := labels[accountIdIdx] + labels[accountLabelIdx] + regionName + m.AccountID + labels[namespaceIdx] + fmt.Sprint("-", valI) + identifier := labels[aws.LabelConst.AccountIdIdx] + labels[aws.LabelConst.AccountLabelIdx] + regionName + m.MonitoringAccountID + labels[aws.LabelConst.NamespaceIdx] + fmt.Sprint("-", valI) if _, ok := events[identifier]; !ok { - if labels[accountIdIdx] != "" { - events[identifier] = aws.InitEvent(regionName, labels[accountLabelIdx], labels[accountIdIdx], output.Timestamps[valI]) + if labels[aws.LabelConst.AccountIdIdx] != "" { + events[identifier] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI]) } else { - events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, output.Timestamps[valI]) + events[identifier] = aws.InitEvent(regionName, m.MonitoringAccountName, m.MonitoringAccountID, output.Timestamps[valI]) } } events[identifier] = insertRootFields(events[identifier], metricDataResultValue, labels) continue } - identifierValue := labels[identifierValueIdx] + identifierValue := labels[aws.LabelConst.IdentifierValueIdx] uniqueIdentifierValue := identifierValue + fmt.Sprint("-", valI) // add tags to event based on identifierValue @@ -580,7 +571,7 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatch.GetMetricDataAPIClient if len(tagsFilter) != 0 && resourceTagMap[subIdentifier] == nil { continue } - events[uniqueIdentifierValue] = aws.InitEvent(regionName, labels[accountLabelIdx], labels[accountIdIdx], output.Timestamps[valI]) + events[uniqueIdentifierValue] = aws.InitEvent(regionName, labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI]) } events[uniqueIdentifierValue] = insertRootFields(events[uniqueIdentifierValue], metricDataResultValue, labels) insertTags(events, uniqueIdentifierValue, subIdentifier, resourceTagMap) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 596b290587ad..ba882d529468 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -776,7 +776,7 @@ func TestGenerateFieldName(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { - fieldName := generateFieldName(c.label[namespaceIdx], c.label) + fieldName := generateFieldName(c.label[aws.LabelConst.NamespaceIdx], c.label) assert.Equal(t, c.expectedFieldName, fieldName) }) } @@ -1288,7 +1288,7 @@ func TestCreateEventsWithIdentifier(t *testing.T) { func TestCreateEventsWithoutIdentifier(t *testing.T) { m := MetricSet{} m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} - m.MetricSet = &aws.MetricSet{Period: 5, AccountID: accountID} + m.MetricSet = &aws.MetricSet{Period: 5, MonitoringAccountID: accountID} m.logger = logp.NewLogger("test") mockTaggingSvc := &MockResourceGroupsTaggingClient{} @@ -1333,7 +1333,7 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { func TestCreateEventsWithDataGranularity(t *testing.T) { m := MetricSet{} m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} - m.MetricSet = &aws.MetricSet{Period: 10, AccountID: accountID, DataGranularity: 5} + m.MetricSet = &aws.MetricSet{Period: 10, MonitoringAccountID: accountID, DataGranularity: 5} m.logger = logp.NewLogger("test") mockTaggingSvc := &MockResourceGroupsTaggingClient{} @@ -1374,7 +1374,7 @@ func TestCreateEventsWithDataGranularity(t *testing.T) { func TestCreateEventsWithTagsFilter(t *testing.T) { m := MetricSet{} m.CloudwatchConfigs = []Config{{Statistic: []string{"Average"}}} - m.MetricSet = &aws.MetricSet{Period: 5, AccountID: accountID} + m.MetricSet = &aws.MetricSet{Period: 5, MonitoringAccountID: accountID} m.logger = logp.NewLogger("test") mockTaggingSvc := &MockResourceGroupsTaggingClient{} @@ -1535,7 +1535,7 @@ func TestCreateEventsTimestamp(t *testing.T) { m := MetricSet{ logger: logp.NewLogger("test"), CloudwatchConfigs: []Config{{Statistic: []string{"Average"}}}, - MetricSet: &aws.MetricSet{Period: 5, AccountID: accountID}, + MetricSet: &aws.MetricSet{Period: 5, MonitoringAccountID: accountID}, } listMetricWithStatsTotal := []metricsWithStatistics{ diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index e0bc5289c478..caf695f1cb9f 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -51,13 +51,13 @@ type MetricWithID struct { // API call per metric name and set of dimensions. This will increase API cost. // IncludeLinkedAccounts is set to true for ListMetrics API to include metrics from source accounts in addition to the // monitoring account. -func GetListMetricsOutput(namespace string, regionName string, period time.Duration, svcCloudwatch cloudwatch.ListMetricsAPIClient) ([]MetricWithID, error) { +func GetListMetricsOutput(namespace string, regionName string, period time.Duration, includeLinkedAccounts bool, monitoringAccountID string, svcCloudwatch cloudwatch.ListMetricsAPIClient) ([]MetricWithID, error) { var metricWithAccountID []MetricWithID var nextToken *string listMetricsInput := &cloudwatch.ListMetricsInput{ NextToken: nextToken, - IncludeLinkedAccounts: true, + IncludeLinkedAccounts: includeLinkedAccounts, } // To filter the results to show only metrics that have had data points published @@ -80,6 +80,14 @@ func GetListMetricsOutput(namespace string, regionName string, period time.Durat return metricWithAccountID, fmt.Errorf("error ListMetrics with Paginator, skipping region %s: %w", regionName, err) } + // when IncludeLinkedAccounts is set to false, ListMetrics API does not return any OwningAccounts + if page.OwningAccounts == nil { + for _, metric := range page.Metrics { + metricWithAccountID = append(metricWithAccountID, MetricWithID{metric, monitoringAccountID}) + } + return metricWithAccountID, nil + } + for i, metric := range page.Metrics { metricWithAccountID = append(metricWithAccountID, MetricWithID{metric, page.OwningAccounts[i]}) } diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index 3d082858b414..5fe62b5c80cc 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -123,9 +123,6 @@ func (m *MockCloudWatchClient) ListMetrics(context.Context, *cloudwatch.ListMetr Dimensions: []cloudwatchtypes.Dimension{dim1}, }, }, - OwningAccounts: []string{ - "1234", - }, NextToken: awssdk.String(""), }, nil } @@ -201,7 +198,7 @@ func (m *MockResourceGroupsTaggingClient) GetResources(_ context.Context, _ *res func TestGetListMetricsOutput(t *testing.T) { svcCloudwatch := &MockCloudWatchClient{} - listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, svcCloudwatch) + listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, false, "123", svcCloudwatch) assert.NoError(t, err) assert.Equal(t, 1, len(listMetricsOutput)) assert.Equal(t, namespace, *listMetricsOutput[0].Metric.Namespace) @@ -213,7 +210,7 @@ func TestGetListMetricsOutput(t *testing.T) { func TestGetListMetricsCrossAccountsOutput(t *testing.T) { svcCloudwatch := &MockCloudwatchClientCrossAccounts{} - listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, svcCloudwatch) + listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, true, "123", svcCloudwatch) assert.NoError(t, err) assert.Equal(t, 2, len(listMetricsOutput)) assert.Equal(t, namespace, *listMetricsOutput[0].Metric.Namespace) @@ -226,7 +223,7 @@ func TestGetListMetricsCrossAccountsOutput(t *testing.T) { func TestGetListMetricsOutputWithWildcard(t *testing.T) { svcCloudwatch := &MockCloudWatchClient{} - listMetricsOutput, err := GetListMetricsOutput("*", "us-west-1", time.Minute*5, svcCloudwatch) + listMetricsOutput, err := GetListMetricsOutput("*", "us-west-1", time.Minute*5, false, "123", svcCloudwatch) assert.NoError(t, err) assert.Equal(t, 1, len(listMetricsOutput)) assert.Equal(t, namespace, *listMetricsOutput[0].Metric.Namespace) diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 8841991b7be8..ddd36a4c3269 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -52,5 +52,6 @@ - module: aws period: 1m latency: 5m + include_linked_accounts: false metricsets: - s3_request