[awsfirehose] Add routing rules for metrics ingested from Firehose

packages/awsfirehose/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.1.0"
+  changes:
+    - description: Add routing rules for metrics from Firehose.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9916
 - version: "1.0.0"
   changes:
     - description: Release package as GA.

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json

@@ -0,0 +1,24 @@
+{
+  "events": [{
+    "@timestamp":"2024-05-01T08:24:00Z",
+    "start_timestamp":"2024-05-01T08:23:00Z",
+    "agent.type":"firehose",
+    "cloud.provider":"aws",
+    "cloud.account.id":"12345678901",
+    "cloud.region":"ap-southeast-2",
+    "aws.exporter.arn":"arn:aws:cloudwatch:ap-southeast-2:12345678901:metric-stream/test-firehose-metrics",
+    "aws.cloudwatch.namespace":"AWS/ApiGateway",
+    "aws.dimensions":{
+      "ApiId": "6am7mj7jqx"
+    },
+    "aws.apigateway.metrics.4XXError":{
+      "count":1,
+      "sum":0.0,
+      "min":0.0,
+      "max":0.0
+    },
+    "data_stream.type":"metrics",
+    "data_stream.dataset":"aws.cloudwatch",
+    "data_stream.namespace":"default"
+  }]
+}

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json-expected.json

@@ -0,0 +1,32 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2024-05-01T08:24:00Z",
+            "agent.type": "firehose",
+            "aws.apigateway.metrics.4XXError": {
+                "count": 1,
+                "max": 0.0,
+                "min": 0.0,
+                "sum": 0.0
+            },
+            "aws.cloudwatch.namespace": "AWS/ApiGateway",
+            "aws.dimensions": {
+                "ApiId": "6am7mj7jqx"
+            },
+            "aws.exporter.arn": "arn:aws:cloudwatch:ap-southeast-2:12345678901:metric-stream/test-firehose-metrics",
+            "cloud": {
+                "provider": "aws"
+            },
+            "cloud.account.id": "12345678901",
+            "cloud.provider": "aws",
+            "cloud.region": "ap-southeast-2",
+            "data_stream.dataset": "aws.apigateway_metrics",
+            "data_stream.namespace": "default",
+            "data_stream.type": "metrics",
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "start_timestamp": "2024-05-01T08:23:00Z"
+        }
+    ]
+}

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-daily-storage-metrics.json

@@ -0,0 +1,48 @@
+{
+  "events": [
+    {
+      "@timestamp":"2024-04-30T08:23:00Z",
+      "start_timestamp":"2024-05-01T08:23:00Z",
+      "agent.type":"firehose",
+      "cloud.provider":"aws",
+      "cloud.account.id":"123456789012",
+      "cloud.region":"us-east-1",
+      "aws.exporter.arn":"arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+      "aws.cloudwatch.namespace":"AWS/S3",
+      "aws.dimensions":{
+        "StorageType":"StandardStorage"
+      },
+      "aws.s3.metrics.BucketSizeBytes":{
+        "count":1,
+        "sum":76353832,
+        "min":76353832,
+        "max":76353832
+      },
+      "data_stream.type":"metrics",
+      "data_stream.dataset":"aws.cloudwatch",
+      "data_stream.namespace":"default"
+    },
+    {
+      "@timestamp":"2024-04-30T08:23:00Z",
+      "start_timestamp":"2024-05-01T08:23:00Z",
+      "agent.type":"firehose",
+      "cloud.provider":"aws",
+      "cloud.account.id":"123456789012",
+      "cloud.region":"us-east-1",
+      "aws.exporter.arn":"arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+      "aws.cloudwatch.namespace":"AWS/S3",
+      "aws.dimensions":{
+        "StorageType":"StandardStorage"
+      },
+      "aws.s3.metrics.NumberOfObjects":{
+        "count":1,
+        "sum":5,
+        "min":5,
+        "max":5
+      },
+      "data_stream.type":"metrics",
+      "data_stream.dataset":"aws.cloudwatch",
+      "data_stream.namespace":"default"
+    }
+  ]
+}

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-daily-storage-metrics.json-expected.json

@@ -0,0 +1,60 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2024-04-30T08:23:00Z",
+            "agent.type": "firehose",
+            "aws.cloudwatch.namespace": "AWS/S3",
+            "aws.dimensions": {
+                "StorageType": "StandardStorage"
+            },
+            "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+            "aws.s3.metrics.BucketSizeBytes": {
+                "count": 1,
+                "max": 76353832,
+                "min": 76353832,
+                "sum": 76353832
+            },
+            "cloud": {
+                "provider": "aws"
+            },
+            "cloud.account.id": "123456789012",
+            "cloud.provider": "aws",
+            "cloud.region": "us-east-1",
+            "data_stream.dataset": "aws.s3_daily_storage",
+            "data_stream.namespace": "default",
+            "data_stream.type": "metrics",
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "start_timestamp": "2024-05-01T08:23:00Z"
+        },
+        {
+            "@timestamp": "2024-04-30T08:23:00Z",
+            "agent.type": "firehose",
+            "aws.cloudwatch.namespace": "AWS/S3",
+            "aws.dimensions": {
+                "StorageType": "StandardStorage"
+            },
+            "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+            "aws.s3.metrics.NumberOfObjects": {
+                "count": 1,
+                "max": 5,
+                "min": 5,
+                "sum": 5
+            },
+            "cloud": {
+                "provider": "aws"
+            },
+            "cloud.account.id": "123456789012",
+            "cloud.provider": "aws",
+            "cloud.region": "us-east-1",
+            "data_stream.dataset": "aws.s3_daily_storage",
+            "data_stream.namespace": "default",
+            "data_stream.type": "metrics",
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "start_timestamp": "2024-05-01T08:23:00Z"
+        }
+    ]
+}

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-request-metrics.json

@@ -0,0 +1,27 @@
+{
+  "events": [
+    {
+      "@timestamp": "2024-05-01T08:24:00Z",
+      "start_timestamp": "2024-05-01T08:23:00Z",
+      "agent.type": "firehose",
+      "cloud.provider": "aws",
+      "cloud.account.id": "123456789012",
+      "cloud.region": "us-east-1",
+      "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+      "aws.cloudwatch.namespace": "AWS/Firehose",
+      "aws.dimensions": {
+        "BucketName": "s3-bucket",
+        "StorageType": "StandardStorage"
+      },
+      "aws.s3.metrics.AllRequests": {
+        "count": 1,
+        "sum": 30,
+        "min": 30,
+        "max": 30
+      },
+      "data_stream.type": "metrics",
+      "data_stream.dataset": "aws.cloudwatch",
+      "data_stream.namespace": "default"
+    }
+  ]
+}

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-s3-request-metrics.json-expected.json

@@ -0,0 +1,33 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2024-05-01T08:24:00Z",
+            "agent.type": "firehose",
+            "aws.cloudwatch.namespace": "AWS/Firehose",
+            "aws.dimensions": {
+                "BucketName": "s3-bucket",
+                "StorageType": "StandardStorage"
+            },
+            "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:123456789012:metric-stream/test-s3",
+            "aws.s3.metrics.AllRequests": {
+                "count": 1,
+                "max": 30,
+                "min": 30,
+                "sum": 30
+            },
+            "cloud": {
+                "provider": "aws"
+            },
+            "cloud.account.id": "123456789012",
+            "cloud.provider": "aws",
+            "cloud.region": "us-east-1",
+            "data_stream.dataset": "aws.s3_request",
+            "data_stream.namespace": "default",
+            "data_stream.type": "metrics",
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "start_timestamp": "2024-05-01T08:23:00Z"
+        }
+    ]
+}

packages/awsfirehose/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,13 @@
+---
+description: Pipeline for rerouting metrics stream from Amazon Data Firehose.
+processors:
+  - set:
+      field: ecs.version
+      value: 8.0.0
+  - set:
+      field: cloud.provider
+      value: aws
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"

packages/awsfirehose/data_stream/metrics/fields/ecs.yml

@@ -0,0 +1,26 @@
+- external: ecs
+  name: cloud.account.id
+- external: ecs
+  name: cloud.provider
+- external: ecs
+  name: cloud.region
+- external: ecs
+  name: data_stream.type
+- external: ecs
+  name: data_stream.dataset
+- external: ecs
+  name: data_stream.namespace
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: error.message
+- external: ecs
+  name: event.id
+- external: ecs
+  name: event.original
+- external: ecs
+  name: message
+- external: ecs
+  name: '@timestamp'
+- external: ecs
+  name: agent.type

packages/awsfirehose/data_stream/metrics/fields/fields.yml

@@ -0,0 +1,34 @@
+- name: start_timestamp
+  type: date
+  description: Date/time when the monitoring cycle started.
+- name: aws
+  type: group
+  fields:
+    - name: cloudwatch
+      type: group
+      fields:
+        - name: namespace
+          type: keyword
+          description: The namespace specified when query cloudwatch api.
+    - name: exporter
+      type: group
+      fields:
+        - name: arn
+          type: keyword
+          description: The metric stream ARN.
+    - name: dimensions
+      type: flattened
+      description: |
+        Metric dimensions.
+    - name: dimensions_fingerprint
+      type: keyword
+      description: |
+        Autogenerated ID representing the fingerprint of the aws.dimensions object.
+      dimension: true
+- name: 'aws.*.metrics.*.*'
+  type: object
+  object_type: double
+  object_type_mapping_type: "*"
+  metric_type: gauge
+  description: |
+    Metrics that returned from Cloudwatch API query.

packages/awsfirehose/data_stream/metrics/manifest.yml

@@ -0,0 +1,8 @@
+title: Metrics ingested from Amazon Data Firehose
+type: metrics
+dataset: awsfirehose
+# This integration includes predefined rules that automatically route AWS service logs to the respective integrations.
+# Adding these permissions to ensure the agents have permissions to write data to `metrics-*-*`.
+elasticsearch:
+  dynamic_dataset: true
+  dynamic_namespace: true