diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index 73ac2833693..8a81dff7142 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.11.10-beta.4" + changes: + - description: Add GCP PubSub ingest pipeline + type: enhancement + link: https://github.com/elastic/integrations/pull/4349 - version: "2.11.10-beta.3" changes: - description: Add GCP Storage ingest pipeline diff --git a/packages/gcp/data_stream/pubsub/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/pubsub/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..41c81b4d9b8 --- /dev/null +++ b/packages/gcp/data_stream/pubsub/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,207 @@ +--- +description: Pipeline for parsing GCP PubSub metrics. +processors: + - rename: + field: gcp.metrics.snapshot.backlog.bytes + target_field: gcp.pubsub.snapshot.backlog.bytes + ignore_missing: true + - rename: + field: gcp.metrics.snapshot.backlog_bytes_by_region.bytes + target_field: gcp.pubsub.snapshot.backlog_bytes_by_region.bytes + ignore_missing: true + - rename: + field: gcp.metrics.snapshot.config_updates.count + target_field: gcp.pubsub.snapshot.config_updates.count + ignore_missing: true + - rename: + field: gcp.metrics.snapshot.num_messages.value + target_field: gcp.pubsub.snapshot.num_messages.value + ignore_missing: true + - rename: + field: gcp.metrics.snapshot.num_messages_by_region.value + target_field: gcp.pubsub.snapshot.num_messages_by_region.value + ignore_missing: true + - rename: + field: gcp.metrics.snapshot.oldest_message_age.sec + target_field: gcp.pubsub.snapshot.oldest_message_age.sec + ignore_missing: true + - rename: + field: gcp.metrics.snapshot.oldest_message_age_by_region.sec + target_field: gcp.pubsub.snapshot.oldest_message_age_by_region.sec + ignore_missing: true + - rename: + field: gcp.metrics.subscription.ack_message.count + target_field: gcp.pubsub.subscription.ack_message.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.backlog.bytes + target_field: gcp.pubsub.subscription.backlog.bytes + ignore_missing: true + - rename: + field: gcp.metrics.subscription.byte_cost.bytes + target_field: gcp.pubsub.subscription.byte_cost.bytes + ignore_missing: true + - rename: + field: gcp.metrics.subscription.config_updates.count + target_field: gcp.pubsub.subscription.config_updates.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.dead_letter_message.count + target_field: gcp.pubsub.subscription.dead_letter_message.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.mod_ack_deadline_message.count + target_field: gcp.pubsub.subscription.mod_ack_deadline_message.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.mod_ack_deadline_message_operation.count + target_field: gcp.pubsub.subscription.mod_ack_deadline_message_operation.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.mod_ack_deadline_request.count + target_field: gcp.pubsub.subscription.mod_ack_deadline_request.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.num_outstanding_messages.value + target_field: gcp.pubsub.subscription.num_outstanding_messages.value + ignore_missing: true + - rename: + field: gcp.metrics.subscription.num_undelivered_messages.value + target_field: gcp.pubsub.subscription.num_undelivered_messages.value + ignore_missing: true + - rename: + field: gcp.metrics.subscription.oldest_retained_acked_message_age.sec + target_field: gcp.pubsub.subscription.oldest_retained_acked_message_age.sec + ignore_missing: true + - rename: + field: gcp.metrics.subscription.oldest_retained_acked_message_age_by_region.value + target_field: gcp.pubsub.subscription.oldest_retained_acked_message_age_by_region.value + ignore_missing: true + - rename: + field: gcp.metrics.subscription.oldest_unacked_message_age.sec + target_field: gcp.pubsub.subscription.oldest_unacked_message_age.sec + ignore_missing: true + - rename: + field: gcp.metrics.subscription.oldest_unacked_message_age_by_region.value + target_field: gcp.pubsub.subscription.oldest_unacked_message_age_by_region.value + ignore_missing: true + - rename: + field: gcp.metrics.subscription.pull_ack_message_operation.count + target_field: gcp.pubsub.subscription.pull_ack_message_operation.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.pull_ack_request.count + target_field: gcp.pubsub.subscription.pull_ack_request.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.pull_message_operation.count + target_field: gcp.pubsub.subscription.pull_message_operation.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.pull_request.count + target_field: gcp.pubsub.subscription.pull_request.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.push_request.count + target_field: gcp.pubsub.subscription.push_request.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.retained_acked.bytes + target_field: gcp.pubsub.subscription.retained_acked.bytes + ignore_missing: true + - rename: + field: gcp.metrics.subscription.retained_acked_bytes_by_region.bytes + target_field: gcp.pubsub.subscription.retained_acked_bytes_by_region.bytes + ignore_missing: true + - rename: + field: gcp.metrics.subscription.seek_request.count + target_field: gcp.pubsub.subscription.seek_request.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.sent_message.count + target_field: gcp.pubsub.subscription.sent_message.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.streaming_pull_ack_message_operation.count + target_field: gcp.pubsub.subscription.streaming_pull_ack_message_operation.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.streaming_pull_ack_request.count + target_field: gcp.pubsub.subscription.streaming_pull_ack_request.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.streaming_pull_message_operation.count + target_field: gcp.pubsub.subscription.streaming_pull_message_operation.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.streaming_pull_mod_ack_deadline_message_operation.count + target_field: gcp.pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.streaming_pull_mod_ack_deadline_request.count + target_field: gcp.pubsub.subscription.streaming_pull_mod_ack_deadline_request.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.streaming_pull_response.count + target_field: gcp.pubsub.subscription.streaming_pull_response.count + ignore_missing: true + - rename: + field: gcp.metrics.subscription.unacked_bytes_by_region.bytes + target_field: gcp.pubsub.subscription.unacked_bytes_by_region.bytes + ignore_missing: true + - rename: + field: gcp.metrics.topic.byte_cost.bytes + target_field: gcp.pubsub.topic.byte_cost.bytes + ignore_missing: true + - rename: + field: gcp.metrics.topic.config_updates.count + target_field: gcp.pubsub.topic.config_updates.count + ignore_missing: true + - rename: + field: gcp.metrics.topic.message_sizes.bytes + target_field: gcp.pubsub.topic.message_sizes.bytes + ignore_missing: true + - rename: + field: gcp.metrics.topic.oldest_retained_acked_message_age_by_region.value + target_field: gcp.pubsub.topic.oldest_retained_acked_message_age_by_region.value + ignore_missing: true + - rename: + field: gcp.metrics.topic.oldest_unacked_message_age_by_region.value + target_field: gcp.pubsub.topic.oldest_unacked_message_age_by_region.value + ignore_missing: true + - rename: + field: gcp.metrics.topic.retained_acked_bytes_by_region.bytes + target_field: gcp.pubsub.topic.retained_acked_bytes_by_region.bytes + ignore_missing: true + - rename: + field: gcp.metrics.topic.send_message_operation.count + target_field: gcp.pubsub.topic.send_message_operation.count + ignore_missing: true + - rename: + field: gcp.metrics.topic.send_request.count + target_field: gcp.pubsub.topic.send_request.count + ignore_missing: true + - rename: + field: gcp.metrics.topic.streaming_pull_response.count + target_field: gcp.pubsub.topic.streaming_pull_response.count + ignore_missing: true + - rename: + field: gcp.metrics.topic.unacked_bytes_by_region.bytes + target_field: gcp.pubsub.topic.unacked_bytes_by_region.bytes + ignore_missing: true + - rename: + field: gcp.metrics.subscription.ack_latencies.value + target_field: gcp.pubsub.subscription.ack_latencies.value + ignore_missing: true + - rename: + field: gcp.metrics.subscription.push_request_latencies.value + target_field: gcp.pubsub.subscription.push_request_latencies.value + ignore_missing: true + - remove: + field: + - gcp.metrics + ignore_missing: true +on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml index dc72a86ed57..6e6438d10d1 100644 --- a/packages/gcp/manifest.yml +++ b/packages/gcp/manifest.yml @@ -1,6 +1,6 @@ name: gcp title: Google Cloud Platform -version: "2.11.10-beta.3" +version: "2.11.10-beta.4" release: ga description: Collect logs from Google Cloud Platform with Elastic Agent. type: integration