Add aws.firehose.arn, aws.firehose.request_id and aws.metrics_names_fingerprint fields

packages/awsfirehose/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.2.1"
+  changes:
+    - description: Add aws.firehose.arn and aws.firehose.request_id fields.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11239
 - version: "1.2.0"
   changes:
     - description: Add AWS Bedrock runtime metrics to Firehose rerouting.

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json

@@ -8,6 +8,9 @@
             "cloud.account.id": "12345678901",
             "cloud.region": "ap-southeast-2",
             "aws.exporter.arn": "arn:aws:cloudwatch:ap-southeast-2:12345678901:metric-stream/test-firehose-metrics",
+            "aws.firehose.arn": "arn:aws:firehose:ap-southeast-2:12345678901:deliverystream/KS-PUT-ELI-2",
+            "aws.firehose.parameters.X-Found-Cluster": "bb8e51259abe4c21996954f5cfe90af1",
+            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
             "aws.cloudwatch.namespace": "AWS/ApiGateway",
             "aws.dimensions": {
                 "ApiId": "6am7mj7jqx"

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json-expected.json

@@ -14,6 +14,9 @@
                 "ApiId": "6am7mj7jqx"
             },
             "aws.exporter.arn": "arn:aws:cloudwatch:ap-southeast-2:12345678901:metric-stream/test-firehose-metrics",
+            "aws.firehose.arn": "arn:aws:firehose:ap-southeast-2:12345678901:deliverystream/KS-PUT-ELI-2",
+            "aws.firehose.parameters.X-Found-Cluster": "bb8e51259abe4c21996954f5cfe90af1",
+            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
             "cloud.account.id": "12345678901",
             "cloud.provider": "aws",
             "cloud.region": "ap-southeast-2",

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-rds-metrics.json

@@ -5,9 +5,11 @@
             "start_timestamp": "2024-06-12T21:59:00.000Z",
             "agent.type": "firehose",
             "cloud.provider": "aws",
-            "cloud.account.id": "627286350134",
+            "cloud.account.id": "12345678901",
             "cloud.region": "us-east-1",
-            "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:627286350134:metric-stream/test-all-metrics-ks-1",
+            "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:12345678901:metric-stream/test-all-metrics-ks-1",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-1:12345678901:deliverystream/KS-PUT-ELI-2",
+            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
             "aws.cloudwatch.namespace": "AWS/RDS",
             "aws.dimensions": {
                 "DBInstanceIdentifier": "metricbeat-test-019e245b"

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-rds-metrics.json-expected.json

@@ -7,16 +7,18 @@
             "aws.dimensions": {
                 "DBInstanceIdentifier": "metricbeat-test-019e245b"
             },
-            "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:627286350134:metric-stream/test-all-metrics-ks-1",
+            "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:12345678901:metric-stream/test-all-metrics-ks-1",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-1:12345678901:deliverystream/KS-PUT-ELI-2",
             "aws.firehose.parameters.X-Found-Cluster": "c335685bb4614254a283e02799f09d63",
+            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
             "aws.rds.metrics.FreeStorageSpace": {
                 "avg": 18430775296,
                 "count": 1,
                 "max": 18430775296,
                 "min": 18430775296,
                 "sum": 18430775296
             },
-            "cloud.account.id": "627286350134",
+            "cloud.account.id": "12345678901",
             "cloud.provider": "aws",
             "cloud.region": "us-east-1",
             "data_stream": {

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-daily-storage-metrics.json

@@ -8,6 +8,8 @@
             "cloud.account.id": "123456789012",
             "cloud.region": "us-east-1",
             "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-1:123456789012:deliverystream/KS-PUT-ELI-2",
+            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
             "aws.cloudwatch.namespace": "AWS/S3",
             "aws.dimensions": {
                 "StorageType": "StandardStorage"
@@ -30,6 +32,8 @@
             "cloud.account.id": "123456789012",
             "cloud.region": "us-east-1",
             "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-1:123456789012:deliverystream/KS-PUT-ELI-2",
+            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
             "aws.cloudwatch.namespace": "AWS/S3",
             "aws.dimensions": {
                 "StorageType": "StandardStorage"

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-daily-storage-metrics.json-expected.json

@@ -8,6 +8,8 @@
                 "StorageType": "StandardStorage"
             },
             "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-1:123456789012:deliverystream/KS-PUT-ELI-2",
+            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
             "aws.s3.metrics.BucketSizeBytes": {
                 "count": 1,
                 "max": 76353832,
@@ -33,6 +35,8 @@
                 "StorageType": "StandardStorage"
             },
             "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-1:123456789012:deliverystream/KS-PUT-ELI-2",
+            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
             "aws.s3.metrics.NumberOfObjects": {
                 "count": 1,
                 "max": 5,

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-request-metrics.json

@@ -8,6 +8,8 @@
             "cloud.account.id": "123456789012",
             "cloud.region": "us-east-1",
             "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-1:123456789012:deliverystream/KS-PUT-ELI-2",
+            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
             "aws.cloudwatch.namespace": "AWS/S3",
             "aws.dimensions": {
                 "BucketName": "s3-bucket",

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-request-metrics.json-expected.json

@@ -9,6 +9,8 @@
                 "StorageType": "StandardStorage"
             },
             "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-1:123456789012:deliverystream/KS-PUT-ELI-2",
+            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
             "aws.s3.metrics.AllRequests": {
                 "count": 1,
                 "max": 30,

packages/awsfirehose/data_stream/metrics/fields/fields.yml

@@ -16,7 +16,7 @@
       fields:
         - name: arn
           type: keyword
-          description: The metric stream ARN.
+          description: The metric stream Amazon Resource Name (ARN).
           dimension: true
     - name: dimensions.*
       type: keyword
@@ -26,6 +26,13 @@
     - name: firehose
       type: group
       fields:
+        - name: request_id
+          type: keyword
+          description: HTTP request opaque GUID.
+          dimension: true
+        - name: arn
+          type: keyword
+          description: Amazon Resource Name (ARN) for the firehose stream.
         - name: parameters.*
           type: flattened
           description: |

packages/awsfirehose/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.1.0"
 name: awsfirehose
 title: Amazon Data Firehose
-version: 1.2.0
+version: 1.2.1
 description: Stream logs and metrics from Amazon Data Firehose into Elastic Cloud.
 type: integration
 categories: