From 4a6551a1829c4209121f58286dafe34230f17a31 Mon Sep 17 00:00:00 2001 From: moxarth-rathod Date: Mon, 19 May 2025 15:21:38 +0530 Subject: [PATCH 1/4] [Cribl] Fix handling of metric event type --- packages/cribl/_dev/build/docs/README.md | 2 +- packages/cribl/changelog.yml | 5 ++++ .../elasticsearch/ingest_pipeline/default.yml | 27 +++++++++++++++++++ .../metrics/fields/base-fields.yml | 12 +++++++++ .../cribl/data_stream/metrics/manifest.yml | 8 ++++++ packages/cribl/docs/README.md | 2 +- packages/cribl/manifest.yml | 2 +- 7 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 packages/cribl/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/cribl/data_stream/metrics/fields/base-fields.yml create mode 100644 packages/cribl/data_stream/metrics/manifest.yml diff --git a/packages/cribl/_dev/build/docs/README.md b/packages/cribl/_dev/build/docs/README.md index 6b12ddf1d63..ac3a21618a8 100644 --- a/packages/cribl/_dev/build/docs/README.md +++ b/packages/cribl/_dev/build/docs/README.md @@ -42,6 +42,6 @@ The Cribl integration offers users a way to ingest logs from either of Cribl's E 1. Set **Cloud Id** for the Cloud destination or **Bulk API URLs** for the Elasticsearch destination to point to your Elastic cluster. - 2. Set **Index or Data Stream** to `logs-cribl-default`. + 2. Set **Index or Data Stream** to `logs-cribl-default` for log-type events and to `metrics-cribl-default` for metric-type events. 3. **API key** should be a Base64 encoded Elastic API key, which you can create in Kibana by following the instructions under **Management** > **Stack Management** > **Security** > **API Keys**. If you are using an API key with “Restrict privileges”, be sure to review the Indices privileges to provide at least "auto_configure" and "write" permissions for the logs-* index, which you will be using for these Fleet integration data streams. \ No newline at end of file diff --git a/packages/cribl/changelog.yml b/packages/cribl/changelog.yml index 8aa704894f1..1fe9a8b96c5 100644 --- a/packages/cribl/changelog.yml +++ b/packages/cribl/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.5.1" + changes: + - description: Fix handling of metric event type. + type: bugfix + link: https://github.com/elastic/integrations/pull/1 - version: "0.5.0" changes: - description: Update Kibana constraint to support 9.0.0. diff --git a/packages/cribl/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml b/packages/cribl/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..f87517f732d --- /dev/null +++ b/packages/cribl/data_stream/metrics/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,27 @@ +--- +description: Pipeline for rerouting metric streams from Cribl. +processors: + - set: + field: ecs.version + value: 8.11.0 + - append: + field: tags + value: cribl + allow_duplicates: false + - rename: + field: _raw + target_field: message + ignore_missing: true + # removing id for metric type of event + - remove: + field: _id + tag: remove_id + ignore_missing: true + # The Cribl routing pipeline is managed by Kibana + # https://github.com/elastic/kibana/pull/176439 + - pipeline: + name: 'cribl-routing-pipeline' +on_failure: + - set: + field: error.message + value: "{{{ _ingest.on_failure_message }}}" diff --git a/packages/cribl/data_stream/metrics/fields/base-fields.yml b/packages/cribl/data_stream/metrics/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/cribl/data_stream/metrics/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/cribl/data_stream/metrics/manifest.yml b/packages/cribl/data_stream/metrics/manifest.yml new file mode 100644 index 00000000000..53ee05cd580 --- /dev/null +++ b/packages/cribl/data_stream/metrics/manifest.yml @@ -0,0 +1,8 @@ +title: "Metrics" +type: metrics +dataset: cribl +# This integration includes predefined rules that automatically route AWS service metrics to the respective integrations. +# Adding these permissions to ensure the agents have permissions to write data to `metrics-*-*`. +elasticsearch: + dynamic_dataset: true + dynamic_namespace: true diff --git a/packages/cribl/docs/README.md b/packages/cribl/docs/README.md index 6b12ddf1d63..ac3a21618a8 100644 --- a/packages/cribl/docs/README.md +++ b/packages/cribl/docs/README.md @@ -42,6 +42,6 @@ The Cribl integration offers users a way to ingest logs from either of Cribl's E 1. Set **Cloud Id** for the Cloud destination or **Bulk API URLs** for the Elasticsearch destination to point to your Elastic cluster. - 2. Set **Index or Data Stream** to `logs-cribl-default`. + 2. Set **Index or Data Stream** to `logs-cribl-default` for log-type events and to `metrics-cribl-default` for metric-type events. 3. **API key** should be a Base64 encoded Elastic API key, which you can create in Kibana by following the instructions under **Management** > **Stack Management** > **Security** > **API Keys**. If you are using an API key with “Restrict privileges”, be sure to review the Indices privileges to provide at least "auto_configure" and "write" permissions for the logs-* index, which you will be using for these Fleet integration data streams. \ No newline at end of file diff --git a/packages/cribl/manifest.yml b/packages/cribl/manifest.yml index 5ffc7c3005c..b9418133e93 100644 --- a/packages/cribl/manifest.yml +++ b/packages/cribl/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: cribl title: "Cribl" -version: "0.5.0" +version: "0.5.1" description: Stream logs from Cribl into Elastic. type: integration categories: From c39b91c942d0786d448b0e58bfffe97fb7f634ba Mon Sep 17 00:00:00 2001 From: moxarth-rathod Date: Mon, 19 May 2025 16:09:17 +0530 Subject: [PATCH 2/4] Add PR link in the changelog --- packages/cribl/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cribl/changelog.yml b/packages/cribl/changelog.yml index 1fe9a8b96c5..1406db3eb33 100644 --- a/packages/cribl/changelog.yml +++ b/packages/cribl/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Fix handling of metric event type. type: bugfix - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/13930 - version: "0.5.0" changes: - description: Update Kibana constraint to support 9.0.0. From b2107a537cce73b0d936a434f086b20f99200be7 Mon Sep 17 00:00:00 2001 From: moxarth-rathod <96762084+moxarth-rathod@users.noreply.github.com> Date: Wed, 21 May 2025 10:39:58 +0530 Subject: [PATCH 3/4] fix minor nit Co-authored-by: Kylie Meli --- packages/cribl/_dev/build/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cribl/_dev/build/docs/README.md b/packages/cribl/_dev/build/docs/README.md index ac3a21618a8..dfd130e9230 100644 --- a/packages/cribl/_dev/build/docs/README.md +++ b/packages/cribl/_dev/build/docs/README.md @@ -42,6 +42,6 @@ The Cribl integration offers users a way to ingest logs from either of Cribl's E 1. Set **Cloud Id** for the Cloud destination or **Bulk API URLs** for the Elasticsearch destination to point to your Elastic cluster. - 2. Set **Index or Data Stream** to `logs-cribl-default` for log-type events and to `metrics-cribl-default` for metric-type events. + 2. Set **Index or Data Stream** to `logs-cribl-default` for log-type events or to `metrics-cribl-default` for metric-type events. 3. **API key** should be a Base64 encoded Elastic API key, which you can create in Kibana by following the instructions under **Management** > **Stack Management** > **Security** > **API Keys**. If you are using an API key with “Restrict privileges”, be sure to review the Indices privileges to provide at least "auto_configure" and "write" permissions for the logs-* index, which you will be using for these Fleet integration data streams. \ No newline at end of file From e3bd28713a5eb59c3bdba5ef6491ab22ed8d077a Mon Sep 17 00:00:00 2001 From: moxarth-rathod Date: Fri, 23 May 2025 10:54:10 +0530 Subject: [PATCH 4/4] Fix CI --- packages/cribl/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cribl/docs/README.md b/packages/cribl/docs/README.md index ac3a21618a8..dfd130e9230 100644 --- a/packages/cribl/docs/README.md +++ b/packages/cribl/docs/README.md @@ -42,6 +42,6 @@ The Cribl integration offers users a way to ingest logs from either of Cribl's E 1. Set **Cloud Id** for the Cloud destination or **Bulk API URLs** for the Elasticsearch destination to point to your Elastic cluster. - 2. Set **Index or Data Stream** to `logs-cribl-default` for log-type events and to `metrics-cribl-default` for metric-type events. + 2. Set **Index or Data Stream** to `logs-cribl-default` for log-type events or to `metrics-cribl-default` for metric-type events. 3. **API key** should be a Base64 encoded Elastic API key, which you can create in Kibana by following the instructions under **Management** > **Stack Management** > **Security** > **API Keys**. If you are using an API key with “Restrict privileges”, be sure to review the Indices privileges to provide at least "auto_configure" and "write" permissions for the logs-* index, which you will be using for these Fleet integration data streams. \ No newline at end of file