[AWS] Add include_linked_accounts parameter for all metrics data streams

packages/aws/_dev/build/docs/README.md

@@ -55,6 +55,18 @@ The extra-charges generated by GetMetricData API calls are proportional to the f
 * `Collection Period` and `Data Granularity`. By setting `Collection Period` and `Data Granularity` together, you can control, respectively, how frequently you want your metrics to be collected and how granular they have to be. If you can tolerate an extra delay in retrieving metrics as trade off, you may consider setting `data_granularity` and increase the value for `Collection Period` to reduce extra charges. For example, setting `Data Granularity` to your current value for `Period`, and doubling the value of `Period`, may lead to a 50% savings.
 * `Tags Filter`. By specifying a tag, you can ensure that no Cloudwatch API call is performed for AWS resources you are not interested in.
 
+### Cross-account observability
+The `include_linked_accounts` parameter is used to enable the inclusion of metrics from different accounts linked to a 
+main monitoring account. By setting this parameter to true, users can gather metrics from multiple AWS accounts that are
+linked through the [CloudWatch cross-account observability](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account.html).
+By default, the `include_linked_accounts` parameter is set to false, meaning that only metrics from the main monitoring
+account are collected. When set to true, the parameter allows the CloudWatch ListMetrics API to include metrics from
+the monitoring account and all linked source accounts in the returned data, providing a comprehensive cross-account view.
+
+*_Note_:* Users should ensure that the necessary IAM roles and policies are properly set up in order to link the monitoring
+account and source accounts together. 
+Please see [Link monitoring accounts with source accounts](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account-Setup.html#CloudWatch-Unified-Cross-Account-Setup-permissions) for more details.
+
 ## Requirements
 
 Before using the AWS integration you will need:

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.43.0"
+  changes:
+    - description: Add include_linked_accounts config parameter for metrics data streams.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6459
 - version: "1.42.0"
   changes:
     - description: Add field agent.id to be set as dimension for TSDB migration.

packages/aws/data_stream/billing/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/billing/manifest.yml

@@ -41,5 +41,13 @@ streams:
         show_user: true
         default:
           - "aws:createdBy"
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
     title: AWS Billing Metrics
     description: Collect billing metrics from Amazon Web Services with Elastic Agent.

packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/cloudwatch_metrics/manifest.yml

@@ -17,6 +17,14 @@ streams:
         multi: false
         required: false
         show_user: false
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
       - name: regions
         type: text
         title: Regions

packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/dynamodb/manifest.yml

@@ -37,5 +37,13 @@ streams:
         default: |
           # - key: "created-by"
             # value: "foo"
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
     title: AWS DynamoDB metrics
     description: Collect AWS DynamoDB metrics

packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/ebs/manifest.yml

@@ -37,5 +37,13 @@ streams:
         default: |
           # - key: "created-by"
             # value: "foo"
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
     title: AWS EBS metrics
     description: Collect AWS EBS metrics

packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/ec2_metrics/manifest.yml

@@ -37,5 +37,13 @@ streams:
         default: |
           # - key: "created-by"
             # value: "foo"
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
     title: AWS EC2 metrics
     description: Collect AWS EC2 metrics

packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/ecs_metrics/manifest.yml

@@ -37,5 +37,13 @@ streams:
         default: |
           # - key: "created-by"
             # value: "foo"
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
     title: AWS ECS metrics
     description: Collect AWS ECS metrics

packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/elb_metrics/manifest.yml

@@ -37,5 +37,13 @@ streams:
         default: |
           # - key: "created-by"
             # value: "foo"
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
     title: AWS ELB metrics
     description: Collect AWS ELB metrics

packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/firewall_metrics/manifest.yml

@@ -28,6 +28,14 @@ streams:
         multi: false
         required: false
         show_user: false
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
       - name: metrics
         type: yaml
         title: Metrics

packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/kinesis/manifest.yml

@@ -37,5 +37,13 @@ streams:
         default: |
           # - key: "created-by"
             # value: "foo"
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
     title: AWS Kinesis Data Stream metrics
     description: Collect AWS Kinesis Data Stream metrics

packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/lambda/manifest.yml

@@ -37,5 +37,13 @@ streams:
         default: |
           # - key: "created-by"
             # value: "foo"
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
     title: AWS Lambda metrics
     description: Collect AWS Lambda metrics

packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/natgateway/manifest.yml

@@ -28,5 +28,13 @@ streams:
         multi: false
         required: false
         show_user: false
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: false
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is false.
     title: AWS NAT gateway metrics
     description: Collect AWS NAT gateway metrics

packages/aws/data_stream/rds/agent/stream/stream.yml.hbs

@@ -3,6 +3,9 @@ period: {{period}}
 {{#if data_granularity}}
 data_granularity: {{data_granularity}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
 {{/if}}

packages/aws/data_stream/rds/fields/ecs.yml

@@ -64,4 +64,4 @@
   name: container.name
 - name: agent.id
   external: ecs
-  dimension: true
+  dimension: true

packages/aws/data_stream/rds/fields/fields.yml

@@ -50,13 +50,11 @@
               type: long
               description: |
                 The number of earned CPU credits that an instance has accrued since it was launched or started.
-
         - name: database_connections
           type: long
           metric_type: gauge
           description: |
             The number of database connections in use.
-
         - name: db_instance
           type: group
           fields:
@@ -89,8 +87,6 @@
               type: keyword
               description: |
                 Each DB instance runs a DB engine, like MySQL, MariaDB, PostgreSQL and etc.
-
-
         - name: disk_queue_depth
           metric_type: gauge
           type: float
@@ -123,7 +119,6 @@
           type: long
           description: |
             The lagging size of the replica lagging the most in terms of WAL data received. Applies to PostgreSQL.
-
         - name: read_io.ops_per_sec
           metric_type: gauge
           type: float
@@ -190,7 +185,6 @@
           type: long
           description: |
             The average number of failed login attempts per second.
-
         - name: throughput
           type: group
           fields:
@@ -254,7 +248,6 @@
               type: float
               description: |
                 The average number of bytes written to disk per second.
-
         - name: latency
           type: group
           fields:
@@ -312,7 +305,6 @@
               format: duration
               description: |
                 The amount of latency for delete queries, in milliseconds.
-
         - name: disk_usage
           type: group
           fields:
@@ -332,7 +324,6 @@
               type: long
               description: |
                 The disk space used by transaction logs. Applies to PostgreSQL.
-
         - name: transactions
           type: group
           fields:
@@ -346,7 +337,6 @@
               type: long
               description: |
                 The average number of transactions in the database that are blocked per second.
-
         - name: aurora_bin_log_replica_lag
           metric_type: gauge
           type: long
@@ -393,7 +383,6 @@
               type: long
               description: |
                 The minimum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds.
-
         - name: backtrack_change_records.creation_rate
           metric_type: gauge
           type: long