[AWS] Add include_linked_accounts config parameter for ListMetrics API

CHANGELOG.next.asciidoc

@@ -325,6 +325,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Add GCP Carbon Footprint metricbeat data {pull}34820[34820]
 - Add event loop utilization metric to Kibana module {pull}35020[35020]
 - Support collecting metrics from both the monitoring account and linked accounts from AWS CloudWatch. {pull}35540[35540]
+- Add new parameter `include_linked_accounts` to enable/disable metrics collection from multiple linked AWS Accounts {pull}35648[35648]
 - Migrate Azure Billing, Monitor, and Storage metricsets to the newer SDK. {pull}33585[33585]
 
 *Osquerybeat*

metricbeat/docs/modules/aws.asciidoc

@@ -73,7 +73,14 @@ services do not include a region. In `aws` module, `endpoint` config is to set
 the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`,
 `sc2s.sgov.gov`.
 
-If endpoint is specified, `regions` config becomes required. For example:
+If endpoint is specified, `regions` config becomes required.
+
+* *include_linked_accounts*
+
+The `include_linked_accounts` parameter is used to enable the inclusion of metrics from different accounts linked to a main monitoring account. By setting this parameter to true, users can gather metrics from multiple AWS accounts that are linked through the https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Cross-Account-Cross-Region.html[Cross-Account Cross-Region feature in Amazon CloudWatch].
+By default, the `include_linked_accounts` parameter is set to false, meaning that only metrics from the main monitoring account are collected. When set to true, the parameter allows the CloudWatch service to access and retrieve metrics from the linked accounts, providing a comprehensive view of metrics across multiple accounts.
+
+*_Note_:* Enabling the include_linked_accounts parameter requires appropriate permissions and authentication credentials for the linked accounts. Users should ensure that the necessary IAM roles and policies are properly set up to establish the necessary trust and access between accounts.
 
 * *tags_filter*
 
@@ -413,6 +420,12 @@ metricbeat.modules:
     - transitgateway
     - usage
     - vpn
+- module: aws
+  period: 1m
+  latency: 5m
+  include_linked_accounts: false
+  metricsets:
+    - s3_request
 ----
 
 [float]

x-pack/metricbeat/metricbeat.reference.yml

@@ -237,6 +237,12 @@ metricbeat.modules:
     - transitgateway
     - usage
     - vpn
+- module: aws
+  period: 1m
+  latency: 5m
+  include_linked_accounts: false
+  metricsets:
+    - s3_request
 
 #----------------------------- AWS Fargate Module -----------------------------
 - module: awsfargate

x-pack/metricbeat/module/aws/_meta/config.reference.yml

@@ -51,3 +51,9 @@
     - transitgateway
     - usage
     - vpn
+- module: aws
+  period: 1m
+  latency: 5m
+  include_linked_accounts: false
+  metricsets:
+    - s3_request

x-pack/metricbeat/module/aws/_meta/config.yml

@@ -49,5 +49,6 @@
 - module: aws
   period: 1m
   latency: 5m
+  include_linked_accounts: false
   metricsets:
     - s3_request

x-pack/metricbeat/module/aws/_meta/docs.asciidoc

@@ -61,7 +61,14 @@ services do not include a region. In `aws` module, `endpoint` config is to set
 the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`,
 `sc2s.sgov.gov`.
 
-If endpoint is specified, `regions` config becomes required. For example:
+If endpoint is specified, `regions` config becomes required.
+
+* *include_linked_accounts*
+
+The `include_linked_accounts` parameter is used to enable the inclusion of metrics from different accounts linked to a main monitoring account. By setting this parameter to true, users can gather metrics from multiple AWS accounts that are linked through the https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Cross-Account-Cross-Region.html[Cross-Account Cross-Region feature in Amazon CloudWatch].
+By default, the `include_linked_accounts` parameter is set to false, meaning that only metrics from the main monitoring account are collected. When set to true, the parameter allows the CloudWatch service to access and retrieve metrics from the linked accounts, providing a comprehensive view of metrics across multiple accounts.
+
+*_Note_:* Enabling the include_linked_accounts parameter requires appropriate permissions and authentication credentials for the linked accounts. Users should ensure that the necessary IAM roles and policies are properly set up to establish the necessary trust and access between accounts.
 
 * *tags_filter*
 

x-pack/metricbeat/module/aws/aws.go

@@ -29,26 +29,28 @@ type describeRegionsClient interface {
 
 // Config defines all required and optional parameters for aws metricsets
 type Config struct {
-	Period          time.Duration       `config:"period" validate:"nonzero,required"`
-	DataGranularity time.Duration       `config:"data_granularity"`
-	Regions         []string            `config:"regions"`
-	Latency         time.Duration       `config:"latency"`
-	AWSConfig       awscommon.ConfigAWS `config:",inline"`
-	TagsFilter      []Tag               `config:"tags_filter"`
+	Period                time.Duration       `config:"period" validate:"nonzero,required"`
+	DataGranularity       time.Duration       `config:"data_granularity"`
+	Regions               []string            `config:"regions"`
+	Latency               time.Duration       `config:"latency"`
+	AWSConfig             awscommon.ConfigAWS `config:",inline"`
+	TagsFilter            []Tag               `config:"tags_filter"`
+	IncludeLinkedAccounts *bool               `config:"include_linked_accounts"`
 }
 
 // MetricSet is the base metricset for all aws metricsets
 type MetricSet struct {
 	mb.BaseMetricSet
-	RegionsList     []string
-	Endpoint        string
-	Period          time.Duration
-	DataGranularity time.Duration
-	Latency         time.Duration
-	AwsConfig       *awssdk.Config
-	AccountName     string
-	AccountID       string
-	TagsFilter      []Tag
+	RegionsList           []string
+	Endpoint              string
+	Period                time.Duration
+	DataGranularity       time.Duration
+	Latency               time.Duration
+	AwsConfig             *awssdk.Config
+	MonitoringAccountName string
+	MonitoringAccountID   string
+	TagsFilter            []Tag
+	IncludeLinkedAccounts bool
 }
 
 // Tag holds a configuration specific for ec2 and cloudwatch metricset.
@@ -60,6 +62,39 @@ type Tag struct {
 // ModuleName is the name of this module.
 const ModuleName = "aws"
 
+// IncludeLinkedAccountsDefault defines if we should include metrics from linked AWS accounts or not. Default is false.
+// More information about cross-account Cloudwatch monitoring can be found at
+// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Cross-Account-Cross-Region.html
+const IncludeLinkedAccountsDefault = false
+
+type LabelConstants struct {
+	AccountIdIdx             int
+	AccountLabelIdx          int
+	MetricNameIdx            int
+	NamespaceIdx             int
+	StatisticIdx             int
+	IdentifierNameIdx        int
+	IdentifierValueIdx       int
+	LabelLengthTotal         int
+	LabelSeparator           string
+	AccountLabel             string
+	BillingDimensionStartIdx int
+}
+
+var LabelConst = LabelConstants{
+	AccountIdIdx:             0,
+	AccountLabelIdx:          1,
+	MetricNameIdx:            2,
+	NamespaceIdx:             3,
+	StatisticIdx:             4,
+	IdentifierNameIdx:        5,
+	IdentifierValueIdx:       6,
+	LabelLengthTotal:         7,
+	LabelSeparator:           "|",
+	AccountLabel:             "${PROP('AccountLabel')}",
+	BillingDimensionStartIdx: 3,
+}
+
 func init() {
 	if err := mb.Registry.AddModule(ModuleName, newModule); err != nil {
 		panic(err)
@@ -112,10 +147,16 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		Endpoint:        config.AWSConfig.Endpoint,
 	}
 
+	metricSet.IncludeLinkedAccounts = IncludeLinkedAccountsDefault
+	if config.IncludeLinkedAccounts != nil {
+		metricSet.IncludeLinkedAccounts = *config.IncludeLinkedAccounts
+	}
+
 	base.Logger().Debug("Metricset level config for period: ", metricSet.Period)
 	base.Logger().Debug("Metricset level config for data granularity: ", metricSet.DataGranularity)
 	base.Logger().Debug("Metricset level config for tags filter: ", metricSet.TagsFilter)
 	base.Logger().Warn("extra charges on AWS API requests will be generated by this metricset")
+	base.Logger().Debug("Metricset level config for including linked accounts: ", metricSet.IncludeLinkedAccounts)
 
 	// If regions in config is not empty, then overwrite the awsConfig.Region
 	if len(config.Regions) > 0 {
@@ -134,8 +175,8 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 	if err != nil {
 		base.Logger().Warn("failed to get caller identity, please check permission setting: ", err)
 	} else {
-		metricSet.AccountID = *outputIdentity.Account
-		base.Logger().Debug("AWS Credentials belong to account ID: ", metricSet.AccountID)
+		metricSet.MonitoringAccountID = *outputIdentity.Account
+		base.Logger().Debug("AWS Credentials belong to monitoring account ID: ", metricSet.MonitoringAccountID)
 	}
 	// Get account name/alias
 	svcIam := iam.NewFromConfig(awsConfig, func(o *iam.Options) {
@@ -144,7 +185,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		}
 
 	})
-	metricSet.AccountName = getAccountName(svcIam, base, metricSet)
+	metricSet.MonitoringAccountName = getAccountName(svcIam, base, metricSet)
 
 	// Construct MetricSet with a full regions list
 	if config.Regions == nil {
@@ -188,23 +229,23 @@ func getAccountName(svc *iam.Client, base mb.BaseMetricSet, metricSet MetricSet)
 	defer cancel()
 	output, err := svc.ListAccountAliases(ctx, &iam.ListAccountAliasesInput{})
 
-	accountName := metricSet.AccountID
+	accountName := metricSet.MonitoringAccountID
 	if err != nil {
 		base.Logger().Warn("failed to list account aliases, please check permission setting: ", err)
 		return accountName
 	}
 
 	// When there is no account alias, account ID will be used as cloud.account.name
 	if len(output.AccountAliases) == 0 {
-		accountName = metricSet.AccountID
-		base.Logger().Debug("AWS Credentials belong to account ID: ", metricSet.AccountID)
+		accountName = metricSet.MonitoringAccountID
+		base.Logger().Debug("AWS Credentials belong to account ID: ", metricSet.MonitoringAccountID)
 		return accountName
 	}
 
 	// There can be more than one aliases for each account, for now we are only
 	// collecting the first one.
 	accountName = output.AccountAliases[0]
-	base.Logger().Debug("AWS Credentials belong to account name: ", metricSet.AccountName)
+	base.Logger().Debug("AWS Credentials belong to account name: ", metricSet.MonitoringAccountName)
 	return accountName
 }
 

x-pack/metricbeat/module/aws/billing/billing.go

@@ -28,9 +28,8 @@ import (
 )
 
 var (
-	metricsetName  = "billing"
-	regionName     = "us-east-1"
-	labelSeparator = "|"
+	metricsetName = "billing"
+	regionName    = "us-east-1"
 
 	// This list is from https://github.com/aws/aws-sdk-go-v2/blob/master/service/costexplorer/api_enums.go#L60-L90
 	supportedDimensionKeys = []string{
@@ -42,11 +41,7 @@ var (
 		"RESERVATION_ID",
 	}
 
-	dateLayout         = "2006-01-02"
-	accountIdIdx       = 0
-	accountLabelIdx    = 1
-	metricDataValueIdx = 2
-	dimensionStartIdx  = 3
+	dateLayout = "2006-01-02"
 )
 
 // init registers the MetricSet with the central registry as soon as the program
@@ -175,7 +170,7 @@ func (m *MetricSet) getCloudWatchBillingMetrics(
 	endTime time.Time) []mb.Event {
 	var events []mb.Event
 	namespace := "AWS/Billing"
-	listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, svcCloudwatch)
+	listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, m.IncludeLinkedAccounts, m.MonitoringAccountID, svcCloudwatch)
 	if err != nil {
 		m.Logger().Error(err.Error())
 		return nil
@@ -198,17 +193,17 @@ func (m *MetricSet) getCloudWatchBillingMetrics(
 			continue
 		}
 		for valI, metricDataResultValue := range output.Values {
-			labels := strings.Split(*output.Label, labelSeparator)
+			labels := strings.Split(*output.Label, aws.LabelConst.LabelSeparator)
 			event := mb.Event{}
-			if labels[accountIdIdx] != "" {
-				event = aws.InitEvent("", labels[accountLabelIdx], labels[accountIdIdx], output.Timestamps[valI])
+			if labels[aws.LabelConst.AccountIdIdx] != "" {
+				event = aws.InitEvent("", labels[aws.LabelConst.AccountLabelIdx], labels[aws.LabelConst.AccountIdIdx], output.Timestamps[valI])
 			} else {
-				event = aws.InitEvent("", m.AccountName, m.AccountID, output.Timestamps[valI])
+				event = aws.InitEvent("", m.MonitoringAccountName, m.MonitoringAccountID, output.Timestamps[valI])
 			}
 
-			_, _ = event.MetricSetFields.Put(labels[metricDataValueIdx], metricDataResultValue)
+			_, _ = event.MetricSetFields.Put(labels[aws.LabelConst.MetricNameIdx], metricDataResultValue)
 
-			i := dimensionStartIdx
+			i := aws.LabelConst.BillingDimensionStartIdx
 			for i < len(labels)-1 {
 				_, _ = event.MetricSetFields.Put(labels[i], labels[i+1])
 				i += 2
@@ -318,7 +313,7 @@ func (m *MetricSet) getCostGroupBy(svcCostExplorer *costexplorer.Client, groupBy
 }
 
 func (m *MetricSet) addCostMetrics(metrics map[string]costexplorertypes.MetricValue, groupDefinition costexplorertypes.GroupDefinition, startDate string, endDate string) mb.Event {
-	event := aws.InitEvent("", m.AccountName, m.AccountID, time.Now())
+	event := aws.InitEvent("", m.MonitoringAccountName, m.MonitoringAccountID, time.Now())
 
 	// add group definition
 	_, _ = event.MetricSetFields.Put("group_definition", mapstr.M{
@@ -367,9 +362,9 @@ func createMetricDataQuery(metric aws.MetricWithID, index int, dataGranularity t
 	metricDims := metric.Metric.Dimensions
 	metricName := *metric.Metric.MetricName
 
-	label := metric.AccountID + labelSeparator + "${PROP('AccountLabel')}" + labelSeparator + metricName + labelSeparator
+	label := strings.Join([]string{metric.AccountID, aws.LabelConst.AccountLabel, metricName}, aws.LabelConst.LabelSeparator)
 	for _, dim := range metricDims {
-		label += *dim.Name + labelSeparator + *dim.Value + labelSeparator
+		label += aws.LabelConst.LabelSeparator + *dim.Name + aws.LabelConst.LabelSeparator + *dim.Value
 	}
 
 	metricDataQuery := types.MetricDataQuery{

x-pack/metricbeat/module/aws/billing/billing_integration_test.go

@@ -24,6 +24,7 @@ func TestFetch(t *testing.T) {
 
 	metricSet := mbtest.NewReportingMetricSetV2Error(t, config)
 	events, errs := mbtest.ReportingFetchV2Error(metricSet)
+	fmt.Println("events = ", events)
 	if len(errs) > 0 {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
 	}