[8.x](backport #40691) [AWS] Support owning account for cross account monitoring

CHANGELOG.next.asciidoc

@@ -195,6 +195,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix Azure Monitor metric timespan to restore Storage Account PT1H metrics {issue}40376[40376] {pull}40367[40367]
 - Remove excessive info-level logs in cgroups setup {pull}40491[40491]
 - Add missing ECS Cloud fields in GCP `metrics` metricset when using `exclude_labels: true` {issue}40437[40437] {pull}40467[40467]
+- Add AWS OwningAccount support for cross account monitoring {issue}40570[40570] {pull}40691[40691]
 
 *Osquerybeat*
 

metricbeat/docs/modules/aws.asciidoc

@@ -80,14 +80,29 @@ If endpoint is specified, `regions` config becomes required.
 The `include_linked_accounts` parameter is used to enable the inclusion of metrics from different accounts linked to a
 main monitoring account. By setting this parameter to true, users can gather metrics from multiple AWS accounts that are
 linked through the https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account.html[CloudWatch cross-account observability].
-By default, the `include_linked_accounts` parameter is set to true, meaning that only metrics from the main monitoring
+By default, the `include_linked_accounts` parameter is set to true, meaning that metrics from the main monitoring
 account and all linked accounts are all collected. When set to false, the parameter allows the CloudWatch service to
 only retrieve metrics from the monitoring account.
 
+If you need to collect metrics from a specific linked account, use `owning_account` configuration.
+
 *_Note_:* Users should ensure that the necessary IAM roles and policies are properly set up in order to link the monitoring
 account and source accounts together.
 Please see https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account-Setup.html#CloudWatch-Unified-Cross-Account-Setup-permissions[Link monitoring accounts with source accounts] for more details.
 
+* *owning_account*
+
+This configuration works together with `include_linked_accounts` configuration and allows to collect metrics from a specific linked account.
+The configuration accepts a valid account ID and internally it maps to the `OwningAccount` parameter of https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html[ListMetrics API].
+
+Note that, `include_linked_accounts` should be enabled (which is the default value) to use this parameter.
+
+[source,yaml]
+----
+include_linked_accounts: true
+owning_account: 111111111111
+----
+
 * *tags_filter*
 
 The tags to filter against. If tags are given in config, then only
@@ -401,6 +416,8 @@ metricbeat.modules:
   credential_profile_name: test-mb
   metricsets:
     - cloudwatch
+  include_linked_accounts: true
+  # owning_account: 111111111111
   metrics:
     - namespace: AWS/EC2
       name: ["CPUUtilization"]

x-pack/metricbeat/metricbeat.reference.yml

@@ -227,6 +227,8 @@ metricbeat.modules:
   credential_profile_name: test-mb
   metricsets:
     - cloudwatch
+  include_linked_accounts: true
+  # owning_account: 111111111111
   metrics:
     - namespace: AWS/EC2
       name: ["CPUUtilization"]

x-pack/metricbeat/module/aws/_meta/config.reference.yml

@@ -26,6 +26,8 @@
   credential_profile_name: test-mb
   metricsets:
     - cloudwatch
+  include_linked_accounts: true
+  # owning_account: 111111111111
   metrics:
     - namespace: AWS/EC2
       name: ["CPUUtilization"]

x-pack/metricbeat/module/aws/_meta/config.yml

@@ -12,6 +12,8 @@
   period: 5m
   metricsets:
     - cloudwatch
+  include_linked_accounts: true
+  # owning_account: 111111111111
   metrics:
     - namespace: AWS/EC2
       #name: ["CPUUtilization", "DiskWriteOps"]

x-pack/metricbeat/module/aws/_meta/docs.asciidoc

@@ -68,14 +68,29 @@ If endpoint is specified, `regions` config becomes required.
 The `include_linked_accounts` parameter is used to enable the inclusion of metrics from different accounts linked to a
 main monitoring account. By setting this parameter to true, users can gather metrics from multiple AWS accounts that are
 linked through the https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account.html[CloudWatch cross-account observability].
-By default, the `include_linked_accounts` parameter is set to true, meaning that only metrics from the main monitoring
+By default, the `include_linked_accounts` parameter is set to true, meaning that metrics from the main monitoring
 account and all linked accounts are all collected. When set to false, the parameter allows the CloudWatch service to
 only retrieve metrics from the monitoring account.
 
+If you need to collect metrics from a specific linked account, use `owning_account` configuration.
+
 *_Note_:* Users should ensure that the necessary IAM roles and policies are properly set up in order to link the monitoring
 account and source accounts together.
 Please see https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account-Setup.html#CloudWatch-Unified-Cross-Account-Setup-permissions[Link monitoring accounts with source accounts] for more details.
 
+* *owning_account*
+
+This configuration works together with `include_linked_accounts` configuration and allows to collect metrics from a specific linked account.
+The configuration accepts a valid account ID and internally it maps to the `OwningAccount` parameter of https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html[ListMetrics API].
+
+Note that, `include_linked_accounts` should be enabled (which is the default value) to use this parameter.
+
+[source,yaml]
+----
+include_linked_accounts: true
+owning_account: 111111111111
+----
+
 * *tags_filter*
 
 The tags to filter against. If tags are given in config, then only

x-pack/metricbeat/module/aws/aws.go

@@ -36,6 +36,7 @@ type Config struct {
 	AWSConfig             awscommon.ConfigAWS `config:",inline"`
 	TagsFilter            []Tag               `config:"tags_filter"`
 	IncludeLinkedAccounts *bool               `config:"include_linked_accounts"`
+	OwningAccount         string              `config:"owning_account"`
 }
 
 // MetricSet is the base metricset for all aws metricsets
@@ -51,6 +52,7 @@ type MetricSet struct {
 	MonitoringAccountID   string
 	TagsFilter            []Tag
 	IncludeLinkedAccounts bool
+	OwningAccount         string
 }
 
 // Tag holds a configuration specific for ec2 and cloudwatch metricset.
@@ -158,12 +160,24 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		metricSet.IncludeLinkedAccounts = *config.IncludeLinkedAccounts
 	}
 
+	// IncludeLinkedAccounts & OwningAccount properties are connected.
+	// OwningAccount cannot be set if IncludeLinkedAccounts is set to false
+	if !metricSet.IncludeLinkedAccounts && config.OwningAccount != "" {
+		return nil, fmt.Errorf("include_linked_accounts must be `true` when specifying non-empty owning_account, please correct configurations and try again")
+	}
+
 	base.Logger().Debug("Metricset level config for period: ", metricSet.Period)
 	base.Logger().Debug("Metricset level config for data granularity: ", metricSet.DataGranularity)
 	base.Logger().Debug("Metricset level config for tags filter: ", metricSet.TagsFilter)
-	base.Logger().Warn("extra charges on AWS API requests will be generated by this metricset")
 	base.Logger().Debug("Metricset level config for including linked accounts: ", metricSet.IncludeLinkedAccounts)
 
+	if config.OwningAccount != "" {
+		base.Logger().Debug("Metricset level config for OwningAccount: ", metricSet.OwningAccount)
+		metricSet.OwningAccount = config.OwningAccount
+	}
+
+	base.Logger().Warn("extra charges on AWS API requests will be generated by this metricset")
+
 	// If regions in config is not empty, then overwrite the awsConfig.Region
 	if len(config.Regions) > 0 {
 		awsConfig.Region = config.Regions[0]

x-pack/metricbeat/module/aws/billing/billing.go

@@ -170,7 +170,7 @@ func (m *MetricSet) getCloudWatchBillingMetrics(
 	endTime time.Time) []mb.Event {
 	var events []mb.Event
 	namespace := "AWS/Billing"
-	listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, m.IncludeLinkedAccounts, m.MonitoringAccountID, svcCloudwatch)
+	listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, svcCloudwatch)
 	if err != nil {
 		m.Logger().Error(err.Error())
 		return nil

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go

@@ -178,7 +178,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 		}
 
 		// retrieve all the details for all the metrics available in the current region
-		listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.MonitoringAccountID, svcCloudwatch)
+		listMetricsOutput, err := aws.GetListMetricsOutput("*", regionName, m.Period, m.IncludeLinkedAccounts, m.OwningAccount, m.MonitoringAccountID, svcCloudwatch)
 		if err != nil {
 			m.Logger().Errorf("Error while retrieving the list of metrics for region %s: %w", regionName, err)
 		}

x-pack/metricbeat/module/aws/utils.go

@@ -51,7 +51,10 @@ type MetricWithID struct {
 // API call per metric name and set of dimensions. This will increase API cost.
 // IncludeLinkedAccounts is set to true for ListMetrics API to include metrics from source accounts in addition to the
 // monitoring account.
-func GetListMetricsOutput(namespace string, regionName string, period time.Duration, includeLinkedAccounts bool, monitoringAccountID string, svcCloudwatch cloudwatch.ListMetricsAPIClient) ([]MetricWithID, error) {
+// OwningAccount works alongside IncludeLinkedAccounts as a filter mechanism to extract metrics specific to a linked account.
+func GetListMetricsOutput(namespace string, regionName string, period time.Duration, includeLinkedAccounts bool,
+	owningAccount string, monitoringAccountID string, svcCloudwatch cloudwatch.ListMetricsAPIClient) ([]MetricWithID, error) {
+
 	var metricWithAccountID []MetricWithID
 	var nextToken *string
 
@@ -60,6 +63,10 @@ func GetListMetricsOutput(namespace string, regionName string, period time.Durat
 		IncludeLinkedAccounts: &includeLinkedAccounts,
 	}
 
+	if owningAccount != "" {
+		listMetricsInput.OwningAccount = &owningAccount
+	}
+
 	// To filter the results to show only metrics that have had data points published
 	// in the past three hours, specify this parameter with a value of PT3H.
 	// Please see https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html for more details.
@@ -80,13 +87,14 @@ func GetListMetricsOutput(namespace string, regionName string, period time.Durat
 			return metricWithAccountID, fmt.Errorf("error ListMetrics with Paginator, skipping region %s: %w", regionName, err)
 		}
 
-		// when IncludeLinkedAccounts is set to false, ListMetrics API does not return any OwningAccounts
 		for i, metric := range page.Metrics {
-			owningAccount := monitoringAccountID
-			if page.OwningAccounts != nil {
-				owningAccount = page.OwningAccounts[i]
+			if page.OwningAccounts == nil {
+				// When IncludeLinkedAccounts is set to false, ListMetrics API does not return any OwningAccounts.
+				// Hence, account ID is set to the monitoring account ID
+				metricWithAccountID = append(metricWithAccountID, MetricWithID{metric, monitoringAccountID})
+			} else {
+				metricWithAccountID = append(metricWithAccountID, MetricWithID{metric, page.OwningAccounts[i]})
 			}
-			metricWithAccountID = append(metricWithAccountID, MetricWithID{metric, owningAccount})
 		}
 	}
 	return metricWithAccountID, nil

x-pack/metricbeat/module/aws/utils_test.go

@@ -25,6 +25,9 @@ var (
 	instanceID1 = "i-123"
 	instanceID2 = "i-456"
 
+	accID1 = "456"
+	accID2 = "789"
+
 	id1    = "cpu1"
 	label1 = instanceID1 + " " + metricName
 
@@ -131,7 +134,7 @@ func (m *MockCloudWatchClient) ListMetrics(context.Context, *cloudwatch.ListMetr
 	}, nil
 }
 
-func (m *MockCloudwatchClientCrossAccounts) ListMetrics(context.Context, *cloudwatch.ListMetricsInput, ...func(*cloudwatch.Options)) (*cloudwatch.ListMetricsOutput, error) {
+func (m *MockCloudwatchClientCrossAccounts) ListMetrics(_ context.Context, input *cloudwatch.ListMetricsInput, _ ...func(*cloudwatch.Options)) (*cloudwatch.ListMetricsOutput, error) {
 	dim1 := cloudwatchtypes.Dimension{
 		Name:  &dimName,
 		Value: &instanceID1,
@@ -141,6 +144,13 @@ func (m *MockCloudwatchClientCrossAccounts) ListMetrics(context.Context, *cloudw
 		Value: &instanceID2,
 	}
 
+	outAccID1 := accID1
+	outAccID2 := accID2
+	if input.OwningAccount != nil {
+		outAccID1 = *input.OwningAccount
+		outAccID2 = *input.OwningAccount
+	}
+
 	return &cloudwatch.ListMetricsOutput{
 		Metrics: []cloudwatchtypes.Metric{
 			{
@@ -154,11 +164,8 @@ func (m *MockCloudwatchClientCrossAccounts) ListMetrics(context.Context, *cloudw
 				Dimensions: []cloudwatchtypes.Dimension{dim2},
 			},
 		},
-		OwningAccounts: []string{
-			"123",
-			"456",
-		},
-		NextToken: awssdk.String(""),
+		OwningAccounts: []string{outAccID1, outAccID2},
+		NextToken:      awssdk.String(""),
 	}, nil
 }
 
@@ -247,46 +254,71 @@ func (m *MockResourceGroupsTaggingClient) GetResources(_ context.Context, _ *res
 
 func TestGetListMetricsOutput(t *testing.T) {
 	svcCloudwatch := &MockCloudWatchClient{}
-	listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, false, "123", svcCloudwatch)
+	listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, false, "", "123", svcCloudwatch)
+
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(listMetricsOutput))
-	assert.Equal(t, namespace, *listMetricsOutput[0].Metric.Namespace)
-	assert.Equal(t, metricName, *listMetricsOutput[0].Metric.MetricName)
-	assert.Equal(t, 1, len(listMetricsOutput[0].Metric.Dimensions))
-	assert.Equal(t, dimName, *listMetricsOutput[0].Metric.Dimensions[0].Name)
-	assert.Equal(t, instanceID1, *listMetricsOutput[0].Metric.Dimensions[0].Value)
+
+	m1 := listMetricsOutput[0]
+	assert.Equal(t, "123", m1.AccountID)
+	assert.Equal(t, namespace, *m1.Metric.Namespace)
+	assert.Equal(t, metricName, *m1.Metric.MetricName)
+	assert.Equal(t, 1, len(m1.Metric.Dimensions))
+	assert.Equal(t, dimName, *m1.Metric.Dimensions[0].Name)
+	assert.Equal(t, instanceID1, *m1.Metric.Dimensions[0].Value)
 }
 
 func TestGetListMetricsCrossAccountsOutput(t *testing.T) {
 	svcCloudwatch := &MockCloudwatchClientCrossAccounts{}
-	listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, true, "123", svcCloudwatch)
+	listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, true, "", "123", svcCloudwatch)
+	assert.NoError(t, err)
+	assert.Equal(t, 2, len(listMetricsOutput))
+
+	m1 := listMetricsOutput[0]
+	assert.Equal(t, accID1, m1.AccountID)
+	assert.Equal(t, namespace, *m1.Metric.Namespace)
+	assert.Equal(t, metricName, *m1.Metric.MetricName)
+	assert.Equal(t, 1, len(m1.Metric.Dimensions))
+	assert.Equal(t, dimName, *m1.Metric.Dimensions[0].Name)
+	assert.Equal(t, instanceID1, *m1.Metric.Dimensions[0].Value)
+
+	m2 := listMetricsOutput[1]
+	assert.Equal(t, accID2, m2.AccountID)
+	assert.Equal(t, instanceID2, *m2.Metric.Dimensions[0].Value)
+}
+
+func TestGetListMetricsOfOwningAccount(t *testing.T) {
+	svcCloudwatch := &MockCloudwatchClientCrossAccounts{}
+	owningId := "999"
+
+	listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, true, owningId, "123", svcCloudwatch)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(listMetricsOutput))
-	assert.Equal(t, namespace, *listMetricsOutput[0].Metric.Namespace)
-	assert.Equal(t, metricName, *listMetricsOutput[0].Metric.MetricName)
-	assert.Equal(t, 1, len(listMetricsOutput[0].Metric.Dimensions))
-	assert.Equal(t, dimName, *listMetricsOutput[0].Metric.Dimensions[0].Name)
-	assert.Equal(t, instanceID1, *listMetricsOutput[0].Metric.Dimensions[0].Value)
-	assert.Equal(t, instanceID2, *listMetricsOutput[1].Metric.Dimensions[0].Value)
+
+	assert.Equal(t, owningId, listMetricsOutput[0].AccountID)
+	assert.Equal(t, owningId, listMetricsOutput[1].AccountID)
 }
 
 func TestGetListMetricsOutputWithMultiplePages(t *testing.T) {
 	svcCloudwatch := &MockCloudwatchClientMultiplePages{}
-	listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, false, "123", svcCloudwatch)
+	listMetricsOutput, err := GetListMetricsOutput("AWS/EC2", "us-west-1", time.Minute*5, false, "", "123", svcCloudwatch)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(listMetricsOutput))
 }
 
 func TestGetListMetricsOutputWithWildcard(t *testing.T) {
 	svcCloudwatch := &MockCloudWatchClient{}
-	listMetricsOutput, err := GetListMetricsOutput("*", "us-west-1", time.Minute*5, false, "123", svcCloudwatch)
+	listMetricsOutput, err := GetListMetricsOutput("*", "us-west-1", time.Minute*5, false, "", "123", svcCloudwatch)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(listMetricsOutput))
-	assert.Equal(t, namespace, *listMetricsOutput[0].Metric.Namespace)
-	assert.Equal(t, metricName, *listMetricsOutput[0].Metric.MetricName)
-	assert.Equal(t, 1, len(listMetricsOutput[0].Metric.Dimensions))
-	assert.Equal(t, dimName, *listMetricsOutput[0].Metric.Dimensions[0].Name)
-	assert.Equal(t, instanceID1, *listMetricsOutput[0].Metric.Dimensions[0].Value)
+
+	m1 := listMetricsOutput[0]
+	assert.Equal(t, "123", m1.AccountID)
+	assert.Equal(t, namespace, *m1.Metric.Namespace)
+	assert.Equal(t, metricName, *m1.Metric.MetricName)
+	assert.Equal(t, 1, len(m1.Metric.Dimensions))
+	assert.Equal(t, dimName, *m1.Metric.Dimensions[0].Name)
+	assert.Equal(t, instanceID1, *m1.Metric.Dimensions[0].Value)
 }
 
 func TestGetMetricDataPerRegion(t *testing.T) {

x-pack/metricbeat/modules.d/aws.yml.disabled

@@ -15,6 +15,8 @@
   period: 5m
   metricsets:
     - cloudwatch
+  include_linked_accounts: true
+  # owning_account: 111111111111
   metrics:
     - namespace: AWS/EC2
       #name: ["CPUUtilization", "DiskWriteOps"]