From ba993cba09c7fb5030bc033b4decd8df86f4df63 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Wed, 9 Aug 2023 10:54:54 +0200 Subject: [PATCH 1/4] Remove the accidental extra number_of_workers I added an extra `number_of_workers` advanced configuration setting by mistake while adding it to a group of CloudWatch-based integrations missing it. This change removes the extra definition. I used the later setting description because it contains more details. --- .../generic/agent/stream/aws-cloudwatch.yml.hbs | 3 --- packages/aws_logs/data_stream/generic/manifest.yml | 8 +------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs index c141d81c552..db2cf4f64ae 100644 --- a/packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs @@ -62,9 +62,6 @@ api_sleep: {{ api_sleep }} {{#if latency }} latency: {{ latency }} {{/if}} -{{#if number_of_workers }} -number_of_workers: {{ number_of_workers }} -{{/if}} {{#if credential_profile_name}} credential_profile_name: {{credential_profile_name}} diff --git a/packages/aws_logs/data_stream/generic/manifest.yml b/packages/aws_logs/data_stream/generic/manifest.yml index cd3826ec1f5..1ec2ce0dd3d 100644 --- a/packages/aws_logs/data_stream/generic/manifest.yml +++ b/packages/aws_logs/data_stream/generic/manifest.yml @@ -41,7 +41,7 @@ streams: default: 1 required: false show_user: false - description: Number of workers that will process the log groups with the given log_group_name_prefix. Default value is 1. + description: The number of workers assigned to reading from log groups. Each worker will read log events from one of the log groups matching `log_group_name_prefix`. For example, if `log_group_name_prefix` matches five log groups, then `number_of_workers` should be set to `5`. The default value is `1`. - name: log_streams type: text title: Log Streams @@ -95,12 +95,6 @@ streams: required: false show_user: false description: "The amount of time required for the logs to be available to CloudWatch Logs. Sample values, `1m` or `5m` — see Golang [time.ParseDuration](https://pkg.go.dev/time#ParseDuration) for more details. Latency translates the query's time range to consider the CloudWatch Logs latency. Example: `5m` means that the integration will query CloudWatch to search for logs available 5 minutes ago." - - name: number_of_workers - type: integer - title: Number of workers - required: false - show_user: false - description: The number of workers assigned to reading from log groups. Each worker will read log events from one of the log groups matching `log_group_name_prefix`. For example, if `log_group_name_prefix` matches five log groups, then `number_of_workers` should be set to `5`. The default value is `1`. - name: tags type: text title: Tags From 77d4e0fd3497cd2a74a76a46aa1c5660acd1d3f5 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Wed, 9 Aug 2023 12:01:05 +0200 Subject: [PATCH 2/4] Group S3 options by source I am trying to group the S3 options by source: - sqs queue - bucket - aws bucket arn - non-aws bucket name With this approach, we can define shared options like `number_of_workers` only once. This should streamline the options and avoid duplicated definitions. --- .../generic/agent/stream/aws-s3.yml.hbs | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs b/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs index 74f5880d9cc..5885a4ef730 100644 --- a/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs +++ b/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs @@ -4,6 +4,9 @@ data_stream: pipeline: {{pipeline}} {{/if}} +{{! The aws-s3 input can be configured to read from an SQS queue or an S3 bucket. }} + +{{! start SQS queue }} {{#unless bucket_arn}} {{#unless non_aws_bucket_name}} {{#if queue_url }} @@ -11,50 +14,44 @@ queue_url: {{ queue_url }} {{/if}} {{/unless}} {{/unless}} +{{! end SQS queue }} -{{#unless queue_url}} -{{#unless non_aws_bucket_name}} -{{#if bucket_arn }} -bucket_arn: {{ bucket_arn }} -{{/if}} +{{#unless queue_url}}{{! start S3 bucket }} + +{{! +When using an S3 bucket, you can specify only one of the following options: +- An AWS bucket ARN +- A non-AWS bucket name +}} + +{{! shared S3 options }} {{#if number_of_workers }} number_of_workers: {{ number_of_workers }} {{/if}} + +{{#if bucket_list_prefix }} +bucket_list_prefix: {{ bucket_list_prefix }} +{{/if}} + {{#if bucket_list_interval }} bucket_list_interval: {{ bucket_list_interval }} {{/if}} -{{/unless}} + +{{! AWS S3 bucket ARN options }} +{{#unless non_aws_bucket_name}} +{{#if bucket_arn }} +bucket_arn: {{ bucket_arn }} +{{/if}} {{/unless}} +{{! non-AWS S3 bucket ARN options }} {{#unless bucket_arn}} -{{#unless queue_url}} {{#if non_aws_bucket_name }} non_aws_bucket_name: {{ non_aws_bucket_name }} {{/if}} -{{#if number_of_workers }} -number_of_workers: {{ number_of_workers }} -{{/if}} -{{#if bucket_list_interval }} -bucket_list_interval: {{ bucket_list_interval }} -{{/if}} -{{/unless}} {{/unless}} -{{#unless queue_url}} -{{#unless non_aws_bucket_name}} -{{#unless bucket_arn }} -{{#if bucket_list_prefix }} -bucket_list_prefix: {{ bucket_list_prefix }} -{{/if}} -{{#if number_of_workers }} -number_of_workers: {{ number_of_workers }} -{{/if}} -{{#if bucket_list_interval }} -bucket_list_interval: {{ bucket_list_interval }} -{{/if}} -{{/unless}} -{{/unless}} -{{/unless}} +{{/unless}}{{! end S3 bucket }} {{#if buffer_size }} buffer_size: {{ buffer_size }} From 1662abcaf92a5fedd8385727034cd37be79c8419 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Mon, 21 Aug 2023 18:18:31 +0200 Subject: [PATCH 3/4] Update packages/aws_logs/data_stream/generic/manifest.yml Co-authored-by: Davide Girardi <1390902+girodav@users.noreply.github.com> --- packages/aws_logs/data_stream/generic/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws_logs/data_stream/generic/manifest.yml b/packages/aws_logs/data_stream/generic/manifest.yml index 1ec2ce0dd3d..538fa581622 100644 --- a/packages/aws_logs/data_stream/generic/manifest.yml +++ b/packages/aws_logs/data_stream/generic/manifest.yml @@ -41,7 +41,7 @@ streams: default: 1 required: false show_user: false - description: The number of workers assigned to reading from log groups. Each worker will read log events from one of the log groups matching `log_group_name_prefix`. For example, if `log_group_name_prefix` matches five log groups, then `number_of_workers` should be set to `5`. The default value is `1`. + description: The number of workers assigned to read from log groups. Each worker will read log events from one of the log groups matching `log_group_name_prefix`. For example, if `log_group_name_prefix` matches five log groups, then `number_of_workers` should be set to `5`. The default value is `1`. - name: log_streams type: text title: Log Streams From 19b47e003a953c78acdac99323ec13e48fa1ce9e Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Mon, 21 Aug 2023 18:29:47 +0200 Subject: [PATCH 4/4] Small updates to template comments I am trying to make template more readable. --- .../data_stream/generic/agent/stream/aws-s3.yml.hbs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs b/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs index 5885a4ef730..d43668bf1b2 100644 --- a/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs +++ b/packages/aws_logs/data_stream/generic/agent/stream/aws-s3.yml.hbs @@ -16,7 +16,7 @@ queue_url: {{ queue_url }} {{/unless}} {{! end SQS queue }} -{{#unless queue_url}}{{! start S3 bucket }} +{{#unless queue_url}}{{! start S3 bucket polling }} {{! When using an S3 bucket, you can specify only one of the following options: @@ -24,7 +24,7 @@ When using an S3 bucket, you can specify only one of the following options: - A non-AWS bucket name }} -{{! shared S3 options }} +{{! shared S3 bucket polling options }} {{#if number_of_workers }} number_of_workers: {{ number_of_workers }} {{/if}} @@ -51,7 +51,7 @@ non_aws_bucket_name: {{ non_aws_bucket_name }} {{/if}} {{/unless}} -{{/unless}}{{! end S3 bucket }} +{{/unless}}{{! end S3 bucket polling }} {{#if buffer_size }} buffer_size: {{ buffer_size }}