From 0b403b8698bbee8d948b2513c892f6e625f507c5 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 8 Dec 2021 15:28:55 -0700 Subject: [PATCH 1/6] Add aws-cloudwatch input for log collection in AWS package --- .../agent/stream/aws-cloudwatch.yml.hbs | 48 ++++++++++ .../aws/data_stream/cloudtrail/manifest.yml | 92 ++++++++++++++++++- .../agent/stream/aws-cloudwatch.yml.hbs | 48 ++++++++++ .../data_stream/cloudwatch_logs/manifest.yml | 90 +++++++++++++++++- .../agent/stream/aws-cloudwatch.yml.hbs | 48 ++++++++++ .../aws/data_stream/ec2_logs/manifest.yml | 89 +++++++++++++++++- .../agent/stream/aws-cloudwatch.yml.hbs | 48 ++++++++++ .../aws/data_stream/elb_logs/manifest.yml | 89 +++++++++++++++++- .../agent/stream/aws-cloudwatch.yml.hbs | 48 ++++++++++ .../aws/data_stream/s3access/manifest.yml | 90 +++++++++++++++++- .../agent/stream/aws-cloudwatch.yml.hbs | 48 ++++++++++ packages/aws/data_stream/vpcflow/manifest.yml | 90 +++++++++++++++++- .../waf/agent/stream/aws-cloudwatch.yml.hbs | 48 ++++++++++ packages/aws/data_stream/waf/manifest.yml | 89 +++++++++++++++++- packages/aws/manifest.yml | 50 +++++++--- 15 files changed, 996 insertions(+), 19 deletions(-) create mode 100644 packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs create mode 100644 packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs create mode 100644 packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs create mode 100644 packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs create mode 100644 packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs create mode 100644 packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs create mode 100644 packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..7dba4dfb4d2 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,48 @@ +{{#if log_group_arn}} +log_group_arn: {{log_group_arn}} +{{/if}} +{{#if log_group_name}} +log_group_name: {{log_group_name}} +{{/if}} +{{#if log_group_name_prefix}} +log_group_name_prefix: {{log_group_name_prefix}} +{{/if}} +{{#if region_name}} +region_name: {{region_name}} +{{/if}} +{{#if log_streams}} +log_streams: {{log_streams}} +{{/if}} +{{#if log_stream_prefix}} +log_stream_prefix: {{log_stream_prefix}} +{{/if}} +{{#if start_position}} +start_position: {{start_position}} +{{/if}} +{{#if scan_frequency}} +scan_frequency: {{scan_frequency}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if api_sleep}} +api_sleep: {{api_sleep}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index 2e58cac4234..78a2e724d24 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -3,8 +3,8 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS CloudTrail Logs - description: Collect AWS CloudTrail logs using s3 input + title: AWS CloudTrail Logs via S3 + description: Collect logs using s3 input vars: - name: visibility_timeout type: text @@ -186,3 +186,91 @@ streams: type: bool multi: false default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS CloudTrail Logs via CloudWatch + enabled: false + description: Collect logs using cloudwatch input + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..6310f731989 --- /dev/null +++ b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,48 @@ +{{#if log_group_arn}} +log_group_arn: {{log_group_arn}} +{{/if}} +{{#if log_group_name}} +log_group_name: {{log_group_name}} +{{/if}} +{{#if log_group_name_prefix}} +log_group_name_prefix: {{log_group_name_prefix}} +{{/if}} +{{#if region_name}} +region_name: {{region_name}} +{{/if}} +{{#if log_streams}} +log_streams: {{log_streams}} +{{/if}} +{{#if log_stream_prefix}} +log_stream_prefix: {{log_stream_prefix}} +{{/if}} +{{#if start_position}} +start_position: {{start_position}} +{{/if}} +{{#if scan_frequency}} +scan_frequency: {{scan_frequency}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if api_sleep}} +api_sleep: {{api_sleep}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml index 1e578d3def0..73397ef0812 100644 --- a/packages/aws/data_stream/cloudwatch_logs/manifest.yml +++ b/packages/aws/data_stream/cloudwatch_logs/manifest.yml @@ -3,7 +3,8 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS CloudWatch logs + title: AWS CloudWatch logs via S3 + enabled: false description: Collect AWS CloudWatch logs using s3 input vars: - name: visibility_timeout @@ -61,3 +62,90 @@ streams: type: bool multi: false default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS CloudWatch logs via CloudWatch + description: Collect AWS CloudWatch logs using cloudwatch input. + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..7dba4dfb4d2 --- /dev/null +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,48 @@ +{{#if log_group_arn}} +log_group_arn: {{log_group_arn}} +{{/if}} +{{#if log_group_name}} +log_group_name: {{log_group_name}} +{{/if}} +{{#if log_group_name_prefix}} +log_group_name_prefix: {{log_group_name_prefix}} +{{/if}} +{{#if region_name}} +region_name: {{region_name}} +{{/if}} +{{#if log_streams}} +log_streams: {{log_streams}} +{{/if}} +{{#if log_stream_prefix}} +log_stream_prefix: {{log_stream_prefix}} +{{/if}} +{{#if start_position}} +start_position: {{start_position}} +{{/if}} +{{#if scan_frequency}} +scan_frequency: {{scan_frequency}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if api_sleep}} +api_sleep: {{api_sleep}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml index 33d4699baa2..7ef6bcba4ef 100644 --- a/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/packages/aws/data_stream/ec2_logs/manifest.yml @@ -3,7 +3,7 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS EC2 logs + title: AWS EC2 Logs via S3 description: Collect AWS EC2 logs using s3 input vars: - name: visibility_timeout @@ -61,3 +61,90 @@ streams: type: bool multi: false default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS EC2 Logs via CloudWatch + description: Collect AWS EC2 logs using cloudwatch input. + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..7dba4dfb4d2 --- /dev/null +++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,48 @@ +{{#if log_group_arn}} +log_group_arn: {{log_group_arn}} +{{/if}} +{{#if log_group_name}} +log_group_name: {{log_group_name}} +{{/if}} +{{#if log_group_name_prefix}} +log_group_name_prefix: {{log_group_name_prefix}} +{{/if}} +{{#if region_name}} +region_name: {{region_name}} +{{/if}} +{{#if log_streams}} +log_streams: {{log_streams}} +{{/if}} +{{#if log_stream_prefix}} +log_stream_prefix: {{log_stream_prefix}} +{{/if}} +{{#if start_position}} +start_position: {{start_position}} +{{/if}} +{{#if scan_frequency}} +scan_frequency: {{scan_frequency}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if api_sleep}} +api_sleep: {{api_sleep}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml index 41b7f894b74..a22e787e72f 100644 --- a/packages/aws/data_stream/elb_logs/manifest.yml +++ b/packages/aws/data_stream/elb_logs/manifest.yml @@ -3,7 +3,7 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS ELB logs + title: AWS ELB Logs via S3 description: Collect AWS ELB logs using s3 input vars: - name: visibility_timeout @@ -61,3 +61,90 @@ streams: type: bool multi: false default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS ELB Logs via CloudWatch + description: Collect AWS ELB logs using cloudwatch input. + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..7dba4dfb4d2 --- /dev/null +++ b/packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,48 @@ +{{#if log_group_arn}} +log_group_arn: {{log_group_arn}} +{{/if}} +{{#if log_group_name}} +log_group_name: {{log_group_name}} +{{/if}} +{{#if log_group_name_prefix}} +log_group_name_prefix: {{log_group_name_prefix}} +{{/if}} +{{#if region_name}} +region_name: {{region_name}} +{{/if}} +{{#if log_streams}} +log_streams: {{log_streams}} +{{/if}} +{{#if log_stream_prefix}} +log_stream_prefix: {{log_stream_prefix}} +{{/if}} +{{#if start_position}} +start_position: {{start_position}} +{{/if}} +{{#if scan_frequency}} +scan_frequency: {{scan_frequency}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if api_sleep}} +api_sleep: {{api_sleep}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml index 5d00452b0c1..ca05421eadc 100644 --- a/packages/aws/data_stream/s3access/manifest.yml +++ b/packages/aws/data_stream/s3access/manifest.yml @@ -3,7 +3,7 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS s3access logs + title: AWS S3 Access Logs via S3 description: Collect AWS s3access logs using s3 input vars: - name: visibility_timeout @@ -61,3 +61,91 @@ streams: type: bool multi: false default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS S3 Access Logs via CloudWatch + description: Collect AWS s3access logs using cloudwatch input + enabled: false + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..7dba4dfb4d2 --- /dev/null +++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,48 @@ +{{#if log_group_arn}} +log_group_arn: {{log_group_arn}} +{{/if}} +{{#if log_group_name}} +log_group_name: {{log_group_name}} +{{/if}} +{{#if log_group_name_prefix}} +log_group_name_prefix: {{log_group_name_prefix}} +{{/if}} +{{#if region_name}} +region_name: {{region_name}} +{{/if}} +{{#if log_streams}} +log_streams: {{log_streams}} +{{/if}} +{{#if log_stream_prefix}} +log_stream_prefix: {{log_stream_prefix}} +{{/if}} +{{#if start_position}} +start_position: {{start_position}} +{{/if}} +{{#if scan_frequency}} +scan_frequency: {{scan_frequency}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if api_sleep}} +api_sleep: {{api_sleep}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml index cb5aa3698d5..3b2798974d1 100644 --- a/packages/aws/data_stream/vpcflow/manifest.yml +++ b/packages/aws/data_stream/vpcflow/manifest.yml @@ -3,8 +3,9 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS vpcflow logs + title: AWS VPC Flow Logs via S3 description: Collect AWS vpcflow logs using s3 input + enabled: false vars: - name: visibility_timeout type: text @@ -61,3 +62,90 @@ streams: type: bool multi: false default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS VPC Flow Logs via CloudWatch + description: Collect AWS VPC flow logs using cloudwatch input. + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..7dba4dfb4d2 --- /dev/null +++ b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,48 @@ +{{#if log_group_arn}} +log_group_arn: {{log_group_arn}} +{{/if}} +{{#if log_group_name}} +log_group_name: {{log_group_name}} +{{/if}} +{{#if log_group_name_prefix}} +log_group_name_prefix: {{log_group_name_prefix}} +{{/if}} +{{#if region_name}} +region_name: {{region_name}} +{{/if}} +{{#if log_streams}} +log_streams: {{log_streams}} +{{/if}} +{{#if log_stream_prefix}} +log_stream_prefix: {{log_stream_prefix}} +{{/if}} +{{#if start_position}} +start_position: {{start_position}} +{{/if}} +{{#if scan_frequency}} +scan_frequency: {{scan_frequency}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if api_sleep}} +api_sleep: {{api_sleep}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml index 6142c5853d7..4fa64d36970 100644 --- a/packages/aws/data_stream/waf/manifest.yml +++ b/packages/aws/data_stream/waf/manifest.yml @@ -3,7 +3,7 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS WAF logs + title: AWS WAF logs via S3 description: Collect AWS WAF logs using s3 input vars: - name: visibility_timeout @@ -61,3 +61,90 @@ streams: type: bool multi: false default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS WAF logs via CloudWatch + description: Collect AWS WAF logs using cloudwatch input + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 3a59706ccbc..dcbf58d0496 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.5.1 +version: 1.7.0 license: basic description: Collect logs and metrics from Amazon Web Services with Elastic Agent. type: integration @@ -104,8 +104,12 @@ policy_templates: - security inputs: - type: aws-s3 - title: Collect logs from Cloudtrail service - description: Collecting Cloudtrail logs using aws-s3 input + title: Collect logs from S3 + description: Collecting logs using aws-s3 input + input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting logs using aws-cloudwatch input input_group: logs - type: httpjson title: Collect logs from third-party REST API (experimental) @@ -129,8 +133,12 @@ policy_templates: - cloudwatch_metrics inputs: - type: aws-s3 + title: Collect logs from S3 + description: Collecting logs using aws-s3 input + input_group: logs + - type: aws-cloudwatch title: Collect logs from CloudWatch - description: Collecting logs from CloudWatch using aws-s3 input + description: Collecting logs using aws-cloudwatch input input_group: logs - type: aws/metrics title: Collect metrics from CloudWatch @@ -193,8 +201,12 @@ policy_templates: - ec2_metrics inputs: - type: aws-s3 - title: Collect logs from EC2 service - description: Collecting EC2 logs using aws-s3 input + title: Collect logs from S3 + description: Collecting logs using aws-s3 input + input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting logs using aws-cloudwatch input input_group: logs - type: aws/metrics title: Collect metrics from EC2 service @@ -220,8 +232,12 @@ policy_templates: - network inputs: - type: aws-s3 - title: Collect logs from ELB service - description: Collecting ELB logs using aws-s3 input + title: Collect from CloudWatch + description: Collecting logs from ELB using aws-s3 input + input_group: logs + - type: aws-cloudwatch + title: Collect from CloudWatch + description: Collecting logs from ELB using aws-cloudwatch input input_group: logs - type: aws/metrics title: Collect metrics from ELB service @@ -312,9 +328,13 @@ policy_templates: - security inputs: - type: aws-s3 - title: Collect S3 access logs + title: Collect logs from S3 description: Collecting S3 access logs using aws-s3 input input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting S3 access logs using aws-cloudwatch input + input_group: logs - type: aws/metrics title: Collect metrics from S3 description: Collecting S3 metrics using AWS CloudWatch @@ -437,9 +457,13 @@ policy_templates: - security inputs: - type: aws-s3 - title: Collect VPC Flow logs + title: Collect logs from S3 description: Collecting VPC Flow logs using aws-s3 input input_group: logs + - type: aws-cloudwatch + title: Collect from CloudWatch + description: Collecting VPC Flow logs using aws-cloudwatch input + input_group: logs icons: - src: /img/logo_vpcflow.svg title: AWS VPC logo @@ -472,9 +496,13 @@ policy_templates: - security inputs: - type: aws-s3 - title: Collect WAF logs + title: Collect logs from S3 description: Collecting WAF logs using aws-s3 input input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting WAF logs using aws-cloudwatch input + input_group: logs icons: - src: /img/logo_waf.svg title: AWS VPC logo From 982213d9efcd7083c476b20cdb06ebded1f3f3a7 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 9 Dec 2021 13:32:53 -0700 Subject: [PATCH 2/6] add changelog --- packages/aws/changelog.yml | 5 + .../agent/stream/aws-cloudwatch.yml.hbs | 28 +++ .../aws/data_stream/cloudtrail/manifest.yml | 19 +- .../agent/stream/aws-cloudwatch.yml.hbs | 30 ++- .../data_stream/cloudwatch_logs/manifest.yml | 191 +++++++++-------- .../agent/stream/aws-cloudwatch.yml.hbs | 28 +++ .../aws/data_stream/ec2_logs/manifest.yml | 21 +- .../agent/stream/aws-cloudwatch.yml.hbs | 28 +++ .../aws/data_stream/elb_logs/manifest.yml | 19 ++ .../agent/stream/aws-cloudwatch.yml.hbs | 28 +++ .../aws/data_stream/s3access/manifest.yml | 20 +- .../agent/stream/aws-cloudwatch.yml.hbs | 28 +++ packages/aws/data_stream/vpcflow/manifest.yml | 192 ++++++++++-------- .../waf/agent/stream/aws-cloudwatch.yml.hbs | 28 +++ packages/aws/data_stream/waf/manifest.yml | 21 +- 15 files changed, 507 insertions(+), 179 deletions(-) diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 51a2f6fd2dd..c3e63146f20 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.7.0" + changes: + - description: Add cloudwatch input into AWS package for log collection + type: enhancement + link: https://github.com/elastic/integrations/pull/2323 - version: "1.6.0" changes: - description: Add max_number_of_messages config option to AWS S3 input config. diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs index 7dba4dfb4d2..40f4ee2cff4 100644 --- a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs @@ -1,6 +1,10 @@ +{{#unless log_group_name}} {{#if log_group_arn}} log_group_arn: {{log_group_arn}} {{/if}} +{{/unless}} + +{{#unless log_group_arn}} {{#if log_group_name}} log_group_name: {{log_group_name}} {{/if}} @@ -10,6 +14,8 @@ log_group_name_prefix: {{log_group_name_prefix}} {{#if region_name}} region_name: {{region_name}} {{/if}} +{{/unless}} + {{#if log_streams}} log_streams: {{log_streams}} {{/if}} @@ -28,6 +34,7 @@ api_timeout: {{api_timeout}} {{#if api_sleep}} api_sleep: {{api_sleep}} {{/if}} + {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -40,9 +47,30 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} + +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index 4cb14e58bdf..f3da52fe398 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -5,6 +5,7 @@ streams: template_path: aws-s3.yml.hbs title: AWS CloudTrail Logs via S3 description: Collect logs using s3 input + enabled: false vars: - name: visibility_timeout type: text @@ -196,7 +197,6 @@ streams: - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS CloudTrail Logs via CloudWatch - enabled: false description: Collect logs using cloudwatch input vars: - name: log_group_arn @@ -273,6 +273,23 @@ streams: show_user: false default: 200ms description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - aws-cloudtrail + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: preserve_original_event required: true show_user: true diff --git a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs index 6310f731989..40f4ee2cff4 100644 --- a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -1,6 +1,10 @@ +{{#unless log_group_name}} {{#if log_group_arn}} log_group_arn: {{log_group_arn}} {{/if}} +{{/unless}} + +{{#unless log_group_arn}} {{#if log_group_name}} log_group_name: {{log_group_name}} {{/if}} @@ -10,6 +14,8 @@ log_group_name_prefix: {{log_group_name_prefix}} {{#if region_name}} region_name: {{region_name}} {{/if}} +{{/unless}} + {{#if log_streams}} log_streams: {{log_streams}} {{/if}} @@ -28,6 +34,7 @@ api_timeout: {{api_timeout}} {{#if api_sleep}} api_sleep: {{api_sleep}} {{/if}} + {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -40,9 +47,30 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} -{{/if}} \ No newline at end of file +{{/if}} + +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml index 8de9b8bfaba..3814497d98a 100644 --- a/packages/aws/data_stream/cloudwatch_logs/manifest.yml +++ b/packages/aws/data_stream/cloudwatch_logs/manifest.yml @@ -69,90 +69,107 @@ streams: default: 5 required: false show_user: false -- input: aws-cloudwatch - template_path: aws-cloudwatch.yml.hbs - title: AWS CloudWatch logs via CloudWatch - description: Collect AWS CloudWatch logs using cloudwatch input. - vars: - - name: log_group_arn - type: text - title: Log Group ARN - multi: false - required: false - show_user: true - description: ARN of the log group to collect logs from. - - name: log_group_name - type: text - title: Log Group Name - multi: false - required: false - show_user: false - description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. - - name: log_group_name_prefix - type: text - title: Log Group Name Prefix - multi: false - required: false - show_user: false - description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. - - name: region_name - type: text - title: Region Name - multi: false - required: false - show_user: false - description: Region that the specified log group or log group prefix belongs to. - - name: log_streams - type: text - title: Log Streams - multi: true - required: false - show_user: false - description: A list of strings of log streams names that Filebeat collect log events from. - - name: log_streams_prefix - type: text - title: Log Stream Prefix - multi: false - required: false - show_user: false - description: A string to filter the results to include only log events from log streams that have names starting with this prefix. - - name: start_position - type: text - title: Start Position - multi: false - required: false - default: beginning - show_user: true - description: Allows user to specify if this input should read log files from the beginning or from the end. - - name: scan_frequency - type: text - title: Scan Frequency - multi: false - required: false - show_user: false - default: 1m - description: This config parameter sets how often Filebeat checks for new log events from the specified log group. - - name: api_timeput - type: text - title: API Timeout - multi: false - required: false - show_user: false - default: 120s - description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. - - name: api_sleep - type: text - title: API Sleep - multi: false - required: false - show_user: false - default: 200ms - description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. - - name: preserve_original_event - required: true - show_user: true - title: Preserve original event - description: Preserves a raw copy of the original event, added to the field `event.original` - type: bool - multi: false - default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS CloudWatch logs via CloudWatch + description: Collect AWS CloudWatch logs using cloudwatch input. + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded + - aws-cloudwatch-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs index 7dba4dfb4d2..40f4ee2cff4 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -1,6 +1,10 @@ +{{#unless log_group_name}} {{#if log_group_arn}} log_group_arn: {{log_group_arn}} {{/if}} +{{/unless}} + +{{#unless log_group_arn}} {{#if log_group_name}} log_group_name: {{log_group_name}} {{/if}} @@ -10,6 +14,8 @@ log_group_name_prefix: {{log_group_name_prefix}} {{#if region_name}} region_name: {{region_name}} {{/if}} +{{/unless}} + {{#if log_streams}} log_streams: {{log_streams}} {{/if}} @@ -28,6 +34,7 @@ api_timeout: {{api_timeout}} {{#if api_sleep}} api_sleep: {{api_sleep}} {{/if}} + {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -40,9 +47,30 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} + +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml index dd30b30ec3d..fc4749fa8c0 100644 --- a/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/packages/aws/data_stream/ec2_logs/manifest.yml @@ -5,6 +5,7 @@ streams: template_path: aws-s3.yml.hbs title: AWS EC2 Logs via S3 description: Collect AWS EC2 logs using s3 input + enabled: false vars: - name: visibility_timeout type: text @@ -68,7 +69,7 @@ streams: default: 5 required: false show_user: false -- input: aws-cloudwatch + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS EC2 Logs via CloudWatch description: Collect AWS EC2 logs using cloudwatch input. @@ -147,6 +148,24 @@ streams: show_user: false default: 200ms description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-ec2-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: preserve_original_event required: true show_user: true diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs index 7dba4dfb4d2..40f4ee2cff4 100644 --- a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -1,6 +1,10 @@ +{{#unless log_group_name}} {{#if log_group_arn}} log_group_arn: {{log_group_arn}} {{/if}} +{{/unless}} + +{{#unless log_group_arn}} {{#if log_group_name}} log_group_name: {{log_group_name}} {{/if}} @@ -10,6 +14,8 @@ log_group_name_prefix: {{log_group_name_prefix}} {{#if region_name}} region_name: {{region_name}} {{/if}} +{{/unless}} + {{#if log_streams}} log_streams: {{log_streams}} {{/if}} @@ -28,6 +34,7 @@ api_timeout: {{api_timeout}} {{#if api_sleep}} api_sleep: {{api_sleep}} {{/if}} + {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -40,9 +47,30 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} + +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml index dbb16ec3d64..48af68891eb 100644 --- a/packages/aws/data_stream/elb_logs/manifest.yml +++ b/packages/aws/data_stream/elb_logs/manifest.yml @@ -5,6 +5,7 @@ streams: template_path: aws-s3.yml.hbs title: AWS ELB Logs via S3 description: Collect AWS ELB logs using s3 input + enabled: false vars: - name: visibility_timeout type: text @@ -147,6 +148,24 @@ streams: show_user: false default: 200ms description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-elb-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: preserve_original_event required: true show_user: true diff --git a/packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs index 7dba4dfb4d2..40f4ee2cff4 100644 --- a/packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs @@ -1,6 +1,10 @@ +{{#unless log_group_name}} {{#if log_group_arn}} log_group_arn: {{log_group_arn}} {{/if}} +{{/unless}} + +{{#unless log_group_arn}} {{#if log_group_name}} log_group_name: {{log_group_name}} {{/if}} @@ -10,6 +14,8 @@ log_group_name_prefix: {{log_group_name_prefix}} {{#if region_name}} region_name: {{region_name}} {{/if}} +{{/unless}} + {{#if log_streams}} log_streams: {{log_streams}} {{/if}} @@ -28,6 +34,7 @@ api_timeout: {{api_timeout}} {{#if api_sleep}} api_sleep: {{api_sleep}} {{/if}} + {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -40,9 +47,30 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} + +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml index 5e64718449b..584fbc3624a 100644 --- a/packages/aws/data_stream/s3access/manifest.yml +++ b/packages/aws/data_stream/s3access/manifest.yml @@ -68,7 +68,7 @@ streams: default: 5 required: false show_user: false -- input: aws-cloudwatch + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS S3 Access Logs via CloudWatch description: Collect AWS s3access logs using cloudwatch input @@ -148,6 +148,24 @@ streams: show_user: false default: 200ms description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-s3access + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: preserve_original_event required: true show_user: true diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs index 7dba4dfb4d2..40f4ee2cff4 100644 --- a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs @@ -1,6 +1,10 @@ +{{#unless log_group_name}} {{#if log_group_arn}} log_group_arn: {{log_group_arn}} {{/if}} +{{/unless}} + +{{#unless log_group_arn}} {{#if log_group_name}} log_group_name: {{log_group_name}} {{/if}} @@ -10,6 +14,8 @@ log_group_name_prefix: {{log_group_name_prefix}} {{#if region_name}} region_name: {{region_name}} {{/if}} +{{/unless}} + {{#if log_streams}} log_streams: {{log_streams}} {{/if}} @@ -28,6 +34,7 @@ api_timeout: {{api_timeout}} {{#if api_sleep}} api_sleep: {{api_sleep}} {{/if}} + {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -40,9 +47,30 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} + +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml index c20d28d154c..5ecd3064156 100644 --- a/packages/aws/data_stream/vpcflow/manifest.yml +++ b/packages/aws/data_stream/vpcflow/manifest.yml @@ -69,90 +69,108 @@ streams: default: 5 required: false show_user: false -- input: aws-cloudwatch - template_path: aws-cloudwatch.yml.hbs - title: AWS VPC Flow Logs via CloudWatch - description: Collect AWS VPC flow logs using cloudwatch input. - vars: - - name: log_group_arn - type: text - title: Log Group ARN - multi: false - required: false - show_user: true - description: ARN of the log group to collect logs from. - - name: log_group_name - type: text - title: Log Group Name - multi: false - required: false - show_user: false - description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. - - name: log_group_name_prefix - type: text - title: Log Group Name Prefix - multi: false - required: false - show_user: false - description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. - - name: region_name - type: text - title: Region Name - multi: false - required: false - show_user: false - description: Region that the specified log group or log group prefix belongs to. - - name: log_streams - type: text - title: Log Streams - multi: true - required: false - show_user: false - description: A list of strings of log streams names that Filebeat collect log events from. - - name: log_streams_prefix - type: text - title: Log Stream Prefix - multi: false - required: false - show_user: false - description: A string to filter the results to include only log events from log streams that have names starting with this prefix. - - name: start_position - type: text - title: Start Position - multi: false - required: false - default: beginning - show_user: true - description: Allows user to specify if this input should read log files from the beginning or from the end. - - name: scan_frequency - type: text - title: Scan Frequency - multi: false - required: false - show_user: false - default: 1m - description: This config parameter sets how often Filebeat checks for new log events from the specified log group. - - name: api_timeput - type: text - title: API Timeout - multi: false - required: false - show_user: false - default: 120s - description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. - - name: api_sleep - type: text - title: API Sleep - multi: false - required: false - show_user: false - default: 200ms - description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. - - name: preserve_original_event - required: true - show_user: true - title: Preserve original event - description: Preserves a raw copy of the original event, added to the field `event.original` - type: bool - multi: false - default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS VPC Flow Logs via CloudWatch + description: Collect AWS VPC flow logs using cloudwatch input. + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-vpcflow + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs index 7dba4dfb4d2..40f4ee2cff4 100644 --- a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs @@ -1,6 +1,10 @@ +{{#unless log_group_name}} {{#if log_group_arn}} log_group_arn: {{log_group_arn}} {{/if}} +{{/unless}} + +{{#unless log_group_arn}} {{#if log_group_name}} log_group_name: {{log_group_name}} {{/if}} @@ -10,6 +14,8 @@ log_group_name_prefix: {{log_group_name_prefix}} {{#if region_name}} region_name: {{region_name}} {{/if}} +{{/unless}} + {{#if log_streams}} log_streams: {{log_streams}} {{/if}} @@ -28,6 +34,7 @@ api_timeout: {{api_timeout}} {{#if api_sleep}} api_sleep: {{api_sleep}} {{/if}} + {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -40,9 +47,30 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} {{#if endpoint}} endpoint: {{endpoint}} {{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} + +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml index d37681db282..bc5bdb5c336 100644 --- a/packages/aws/data_stream/waf/manifest.yml +++ b/packages/aws/data_stream/waf/manifest.yml @@ -68,10 +68,11 @@ streams: default: 5 required: false show_user: false -- input: aws-cloudwatch + - input: aws-cloudwatch template_path: aws-cloudwatch.yml.hbs title: AWS WAF logs via CloudWatch description: Collect AWS WAF logs using cloudwatch input + enabled: false vars: - name: log_group_arn type: text @@ -147,6 +148,24 @@ streams: show_user: false default: 200ms description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-waf + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + - name: preserve_original_event required: true show_user: true From bb68200ea9eaaf8615f7dc63ad0b3c99f57ab63d Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 9 Dec 2021 14:16:27 -0700 Subject: [PATCH 3/6] remove aws-cloudwatch input from s3access --- .../agent/stream/aws-cloudwatch.yml.hbs | 76 ------------- .../aws/data_stream/s3access/manifest.yml | 106 ------------------ packages/aws/manifest.yml | 4 - 3 files changed, 186 deletions(-) delete mode 100644 packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs diff --git a/packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs deleted file mode 100644 index 40f4ee2cff4..00000000000 --- a/packages/aws/data_stream/s3access/agent/stream/aws-cloudwatch.yml.hbs +++ /dev/null @@ -1,76 +0,0 @@ -{{#unless log_group_name}} -{{#if log_group_arn}} -log_group_arn: {{log_group_arn}} -{{/if}} -{{/unless}} - -{{#unless log_group_arn}} -{{#if log_group_name}} -log_group_name: {{log_group_name}} -{{/if}} -{{#if log_group_name_prefix}} -log_group_name_prefix: {{log_group_name_prefix}} -{{/if}} -{{#if region_name}} -region_name: {{region_name}} -{{/if}} -{{/unless}} - -{{#if log_streams}} -log_streams: {{log_streams}} -{{/if}} -{{#if log_stream_prefix}} -log_stream_prefix: {{log_stream_prefix}} -{{/if}} -{{#if start_position}} -start_position: {{start_position}} -{{/if}} -{{#if scan_frequency}} -scan_frequency: {{scan_frequency}} -{{/if}} -{{#if api_timeout}} -api_timeout: {{api_timeout}} -{{/if}} -{{#if api_sleep}} -api_sleep: {{api_sleep}} -{{/if}} - -{{#if access_key_id}} -access_key_id: {{access_key_id}} -{{/if}} -{{#if secret_access_key}} -secret_access_key: {{secret_access_key}} -{{/if}} -{{#if session_token}} -session_token: {{session_token}} -{{/if}} -{{#if role_arn}} -role_arn: {{role_arn}} -{{/if}} -{{#if credential_profile_name}} -credential_profile_name: {{credential_profile_name}} -{{/if}} -{{#if shared_credential_file}} -shared_credential_file: {{shared_credential_file}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} -{{#if proxy_url }} -proxy_url: {{proxy_url}} -{{/if}} - -tags: -{{#if preserve_original_event}} -- preserve_original_event -{{/if}} -{{#each tags as |tag i|}} -- {{tag}} -{{/each}} -{{#contains "forwarded" tags}} -publisher_pipeline.disable_host: true -{{/contains}} -{{#if processors}} -processors: -{{processors}} -{{/if}} diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml index 584fbc3624a..ef977fbae1c 100644 --- a/packages/aws/data_stream/s3access/manifest.yml +++ b/packages/aws/data_stream/s3access/manifest.yml @@ -68,109 +68,3 @@ streams: default: 5 required: false show_user: false - - input: aws-cloudwatch - template_path: aws-cloudwatch.yml.hbs - title: AWS S3 Access Logs via CloudWatch - description: Collect AWS s3access logs using cloudwatch input - enabled: false - vars: - - name: log_group_arn - type: text - title: Log Group ARN - multi: false - required: false - show_user: true - description: ARN of the log group to collect logs from. - - name: log_group_name - type: text - title: Log Group Name - multi: false - required: false - show_user: false - description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. - - name: log_group_name_prefix - type: text - title: Log Group Name Prefix - multi: false - required: false - show_user: false - description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. - - name: region_name - type: text - title: Region Name - multi: false - required: false - show_user: false - description: Region that the specified log group or log group prefix belongs to. - - name: log_streams - type: text - title: Log Streams - multi: true - required: false - show_user: false - description: A list of strings of log streams names that Filebeat collect log events from. - - name: log_streams_prefix - type: text - title: Log Stream Prefix - multi: false - required: false - show_user: false - description: A string to filter the results to include only log events from log streams that have names starting with this prefix. - - name: start_position - type: text - title: Start Position - multi: false - required: false - default: beginning - show_user: true - description: Allows user to specify if this input should read log files from the beginning or from the end. - - name: scan_frequency - type: text - title: Scan Frequency - multi: false - required: false - show_user: false - default: 1m - description: This config parameter sets how often Filebeat checks for new log events from the specified log group. - - name: api_timeput - type: text - title: API Timeout - multi: false - required: false - show_user: false - default: 120s - description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. - - name: api_sleep - type: text - title: API Sleep - multi: false - required: false - show_user: false - default: 200ms - description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. - - name: tags - type: text - title: Tags - multi: true - required: true - show_user: false - default: - - forwarded - - aws-s3access - - name: processors - type: yaml - title: Processors - multi: false - required: false - show_user: false - description: > - Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - - - name: preserve_original_event - required: true - show_user: true - title: Preserve original event - description: Preserves a raw copy of the original event, added to the field `event.original` - type: bool - multi: false - default: false diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index dcbf58d0496..e31597cc5c7 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -331,10 +331,6 @@ policy_templates: title: Collect logs from S3 description: Collecting S3 access logs using aws-s3 input input_group: logs - - type: aws-cloudwatch - title: Collect logs from CloudWatch - description: Collecting S3 access logs using aws-cloudwatch input - input_group: logs - type: aws/metrics title: Collect metrics from S3 description: Collecting S3 metrics using AWS CloudWatch From 450376fea8411d760ac5d1aba8279746211003da Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 13 Dec 2021 13:37:21 -0700 Subject: [PATCH 4/6] change elb logs default with s3 input --- packages/aws/data_stream/elb_logs/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml index 48af68891eb..22b780f2137 100644 --- a/packages/aws/data_stream/elb_logs/manifest.yml +++ b/packages/aws/data_stream/elb_logs/manifest.yml @@ -5,7 +5,6 @@ streams: template_path: aws-s3.yml.hbs title: AWS ELB Logs via S3 description: Collect AWS ELB logs using s3 input - enabled: false vars: - name: visibility_timeout type: text @@ -73,6 +72,7 @@ streams: template_path: aws-cloudwatch.yml.hbs title: AWS ELB Logs via CloudWatch description: Collect AWS ELB logs using cloudwatch input. + enabled: false vars: - name: log_group_arn type: text From a9b83be3edd56c6c1d25030c94a8488184674ff0 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 13 Dec 2021 21:33:13 -0700 Subject: [PATCH 5/6] add cloudwatch input for firewall --- .../agent/stream/aws-cloudwatch.yml.hbs | 76 +++++++++++++ .../data_stream/firewall_logs/manifest.yml | 106 ++++++++++++++++++ packages/aws/manifest.yml | 12 +- 3 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..40f4ee2cff4 --- /dev/null +++ b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,76 @@ +{{#unless log_group_name}} +{{#if log_group_arn}} +log_group_arn: {{log_group_arn}} +{{/if}} +{{/unless}} + +{{#unless log_group_arn}} +{{#if log_group_name}} +log_group_name: {{log_group_name}} +{{/if}} +{{#if log_group_name_prefix}} +log_group_name_prefix: {{log_group_name_prefix}} +{{/if}} +{{#if region_name}} +region_name: {{region_name}} +{{/if}} +{{/unless}} + +{{#if log_streams}} +log_streams: {{log_streams}} +{{/if}} +{{#if log_stream_prefix}} +log_stream_prefix: {{log_stream_prefix}} +{{/if}} +{{#if start_position}} +start_position: {{start_position}} +{{/if}} +{{#if scan_frequency}} +scan_frequency: {{scan_frequency}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if api_sleep}} +api_sleep: {{api_sleep}} +{{/if}} + +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} + +tags: +{{#if preserve_original_event}} +- preserve_original_event +{{/if}} +{{#each tags as |tag i|}} +- {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml index 605ffc87124..86b60d82fc0 100644 --- a/packages/aws/data_stream/firewall_logs/manifest.yml +++ b/packages/aws/data_stream/firewall_logs/manifest.yml @@ -61,3 +61,109 @@ streams: type: bool multi: false default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS ELB Logs via CloudWatch + description: Collect AWS ELB logs using cloudwatch input. + enabled: false + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-firewall-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 4c3cdf93f85..eed73570af0 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -232,11 +232,11 @@ policy_templates: - network inputs: - type: aws-s3 - title: Collect from CloudWatch + title: Collect logs from S3 description: Collecting logs from ELB using aws-s3 input input_group: logs - type: aws-cloudwatch - title: Collect from CloudWatch + title: Collect logs from CloudWatch description: Collecting logs from ELB using aws-cloudwatch input input_group: logs - type: aws/metrics @@ -304,9 +304,13 @@ policy_templates: - firewall_metrics inputs: - type: aws-s3 - title: Collect logs from Network Firewall + title: Collect logs from S3 description: Collecting logs from Network Firewall using aws-s3 input input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting logs from Network Firewall using aws-cloudwatch input + input_group: logs - type: aws/metrics title: Collect metrics from Network Firewall description: Collecting metrics from AWS Network Firewall @@ -496,7 +500,7 @@ policy_templates: description: Collecting VPC Flow logs using aws-s3 input input_group: logs - type: aws-cloudwatch - title: Collect from CloudWatch + title: Collect logs from CloudWatch description: Collecting VPC Flow logs using aws-cloudwatch input input_group: logs icons: From 79e122769aec1321af73cc85859e2acf5be75131 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 5 Jan 2022 15:53:43 -0700 Subject: [PATCH 6/6] adjust aws-cloudwatch.yml.hbs --- .../agent/stream/aws-cloudwatch.yml.hbs | 79 +++++++++++-------- .../agent/stream/aws-cloudwatch.yml.hbs | 79 +++++++++++-------- .../agent/stream/aws-cloudwatch.yml.hbs | 79 +++++++++++-------- .../agent/stream/aws-cloudwatch.yml.hbs | 79 +++++++++++-------- .../agent/stream/aws-cloudwatch.yml.hbs | 79 +++++++++++-------- .../agent/stream/aws-cloudwatch.yml.hbs | 79 +++++++++++-------- .../waf/agent/stream/aws-cloudwatch.yml.hbs | 79 +++++++++++-------- 7 files changed, 336 insertions(+), 217 deletions(-) diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs index 40f4ee2cff4..c1576eedae0 100644 --- a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs @@ -1,40 +1,67 @@ {{#unless log_group_name}} -{{#if log_group_arn}} -log_group_arn: {{log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} {{/if}} {{/unless}} +{{/unless}} {{#unless log_group_arn}} -{{#if log_group_name}} -log_group_name: {{log_group_name}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} {{/if}} -{{#if log_group_name_prefix}} -log_group_name_prefix: {{log_group_name_prefix}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} {{/if}} -{{#if region_name}} -region_name: {{region_name}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} {{/if}} {{/unless}} -{{#if log_streams}} -log_streams: {{log_streams}} +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} {{/if}} -{{#if log_stream_prefix}} -log_stream_prefix: {{log_stream_prefix}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} {{/if}} -{{#if start_position}} -start_position: {{start_position}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} {{/if}} -{{#if scan_frequency}} -scan_frequency: {{scan_frequency}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} {{/if}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if api_sleep}} -api_sleep: {{api_sleep}} +{{#if endpoint}} +endpoint: {{endpoint}} {{/if}} - {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -47,25 +74,15 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if credential_profile_name}} -credential_profile_name: {{credential_profile_name}} -{{/if}} -{{#if shared_credential_file}} -shared_credential_file: {{shared_credential_file}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} - tags: {{#if preserve_original_event}} -- preserve_original_event + - preserve_original_event {{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true diff --git a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs index 40f4ee2cff4..c1576eedae0 100644 --- a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -1,40 +1,67 @@ {{#unless log_group_name}} -{{#if log_group_arn}} -log_group_arn: {{log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} {{/if}} {{/unless}} +{{/unless}} {{#unless log_group_arn}} -{{#if log_group_name}} -log_group_name: {{log_group_name}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} {{/if}} -{{#if log_group_name_prefix}} -log_group_name_prefix: {{log_group_name_prefix}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} {{/if}} -{{#if region_name}} -region_name: {{region_name}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} {{/if}} {{/unless}} -{{#if log_streams}} -log_streams: {{log_streams}} +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} {{/if}} -{{#if log_stream_prefix}} -log_stream_prefix: {{log_stream_prefix}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} {{/if}} -{{#if start_position}} -start_position: {{start_position}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} {{/if}} -{{#if scan_frequency}} -scan_frequency: {{scan_frequency}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} {{/if}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if api_sleep}} -api_sleep: {{api_sleep}} +{{#if endpoint}} +endpoint: {{endpoint}} {{/if}} - {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -47,25 +74,15 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if credential_profile_name}} -credential_profile_name: {{credential_profile_name}} -{{/if}} -{{#if shared_credential_file}} -shared_credential_file: {{shared_credential_file}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} - tags: {{#if preserve_original_event}} -- preserve_original_event + - preserve_original_event {{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs index 40f4ee2cff4..c1576eedae0 100644 --- a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -1,40 +1,67 @@ {{#unless log_group_name}} -{{#if log_group_arn}} -log_group_arn: {{log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} {{/if}} {{/unless}} +{{/unless}} {{#unless log_group_arn}} -{{#if log_group_name}} -log_group_name: {{log_group_name}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} {{/if}} -{{#if log_group_name_prefix}} -log_group_name_prefix: {{log_group_name_prefix}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} {{/if}} -{{#if region_name}} -region_name: {{region_name}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} {{/if}} {{/unless}} -{{#if log_streams}} -log_streams: {{log_streams}} +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} {{/if}} -{{#if log_stream_prefix}} -log_stream_prefix: {{log_stream_prefix}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} {{/if}} -{{#if start_position}} -start_position: {{start_position}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} {{/if}} -{{#if scan_frequency}} -scan_frequency: {{scan_frequency}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} {{/if}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if api_sleep}} -api_sleep: {{api_sleep}} +{{#if endpoint}} +endpoint: {{endpoint}} {{/if}} - {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -47,25 +74,15 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if credential_profile_name}} -credential_profile_name: {{credential_profile_name}} -{{/if}} -{{#if shared_credential_file}} -shared_credential_file: {{shared_credential_file}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} - tags: {{#if preserve_original_event}} -- preserve_original_event + - preserve_original_event {{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs index 40f4ee2cff4..c1576eedae0 100644 --- a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -1,40 +1,67 @@ {{#unless log_group_name}} -{{#if log_group_arn}} -log_group_arn: {{log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} {{/if}} {{/unless}} +{{/unless}} {{#unless log_group_arn}} -{{#if log_group_name}} -log_group_name: {{log_group_name}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} {{/if}} -{{#if log_group_name_prefix}} -log_group_name_prefix: {{log_group_name_prefix}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} {{/if}} -{{#if region_name}} -region_name: {{region_name}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} {{/if}} {{/unless}} -{{#if log_streams}} -log_streams: {{log_streams}} +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} {{/if}} -{{#if log_stream_prefix}} -log_stream_prefix: {{log_stream_prefix}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} {{/if}} -{{#if start_position}} -start_position: {{start_position}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} {{/if}} -{{#if scan_frequency}} -scan_frequency: {{scan_frequency}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} {{/if}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if api_sleep}} -api_sleep: {{api_sleep}} +{{#if endpoint}} +endpoint: {{endpoint}} {{/if}} - {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -47,25 +74,15 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if credential_profile_name}} -credential_profile_name: {{credential_profile_name}} -{{/if}} -{{#if shared_credential_file}} -shared_credential_file: {{shared_credential_file}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} - tags: {{#if preserve_original_event}} -- preserve_original_event + - preserve_original_event {{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs index 40f4ee2cff4..c1576eedae0 100644 --- a/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -1,40 +1,67 @@ {{#unless log_group_name}} -{{#if log_group_arn}} -log_group_arn: {{log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} {{/if}} {{/unless}} +{{/unless}} {{#unless log_group_arn}} -{{#if log_group_name}} -log_group_name: {{log_group_name}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} {{/if}} -{{#if log_group_name_prefix}} -log_group_name_prefix: {{log_group_name_prefix}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} {{/if}} -{{#if region_name}} -region_name: {{region_name}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} {{/if}} {{/unless}} -{{#if log_streams}} -log_streams: {{log_streams}} +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} {{/if}} -{{#if log_stream_prefix}} -log_stream_prefix: {{log_stream_prefix}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} {{/if}} -{{#if start_position}} -start_position: {{start_position}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} {{/if}} -{{#if scan_frequency}} -scan_frequency: {{scan_frequency}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} {{/if}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if api_sleep}} -api_sleep: {{api_sleep}} +{{#if endpoint}} +endpoint: {{endpoint}} {{/if}} - {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -47,25 +74,15 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if credential_profile_name}} -credential_profile_name: {{credential_profile_name}} -{{/if}} -{{#if shared_credential_file}} -shared_credential_file: {{shared_credential_file}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} - tags: {{#if preserve_original_event}} -- preserve_original_event + - preserve_original_event {{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs index 40f4ee2cff4..c1576eedae0 100644 --- a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs @@ -1,40 +1,67 @@ {{#unless log_group_name}} -{{#if log_group_arn}} -log_group_arn: {{log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} {{/if}} {{/unless}} +{{/unless}} {{#unless log_group_arn}} -{{#if log_group_name}} -log_group_name: {{log_group_name}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} {{/if}} -{{#if log_group_name_prefix}} -log_group_name_prefix: {{log_group_name_prefix}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} {{/if}} -{{#if region_name}} -region_name: {{region_name}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} {{/if}} {{/unless}} -{{#if log_streams}} -log_streams: {{log_streams}} +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} {{/if}} -{{#if log_stream_prefix}} -log_stream_prefix: {{log_stream_prefix}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} {{/if}} -{{#if start_position}} -start_position: {{start_position}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} {{/if}} -{{#if scan_frequency}} -scan_frequency: {{scan_frequency}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} {{/if}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if api_sleep}} -api_sleep: {{api_sleep}} +{{#if endpoint}} +endpoint: {{endpoint}} {{/if}} - {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -47,25 +74,15 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if credential_profile_name}} -credential_profile_name: {{credential_profile_name}} -{{/if}} -{{#if shared_credential_file}} -shared_credential_file: {{shared_credential_file}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} - tags: {{#if preserve_original_event}} -- preserve_original_event + - preserve_original_event {{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true diff --git a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs index 40f4ee2cff4..c1576eedae0 100644 --- a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs +++ b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs @@ -1,40 +1,67 @@ {{#unless log_group_name}} -{{#if log_group_arn}} -log_group_arn: {{log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} {{/if}} {{/unless}} +{{/unless}} {{#unless log_group_arn}} -{{#if log_group_name}} -log_group_name: {{log_group_name}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} {{/if}} -{{#if log_group_name_prefix}} -log_group_name_prefix: {{log_group_name_prefix}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} {{/if}} -{{#if region_name}} -region_name: {{region_name}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} {{/if}} {{/unless}} -{{#if log_streams}} -log_streams: {{log_streams}} +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} {{/if}} -{{#if log_stream_prefix}} -log_stream_prefix: {{log_stream_prefix}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} {{/if}} -{{#if start_position}} -start_position: {{start_position}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} {{/if}} -{{#if scan_frequency}} -scan_frequency: {{scan_frequency}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} {{/if}} {{#if api_timeout}} api_timeout: {{api_timeout}} {{/if}} -{{#if api_sleep}} -api_sleep: {{api_sleep}} +{{#if endpoint}} +endpoint: {{endpoint}} {{/if}} - {{#if access_key_id}} access_key_id: {{access_key_id}} {{/if}} @@ -47,25 +74,15 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if credential_profile_name}} -credential_profile_name: {{credential_profile_name}} -{{/if}} -{{#if shared_credential_file}} -shared_credential_file: {{shared_credential_file}} -{{/if}} -{{#if endpoint}} -endpoint: {{endpoint}} -{{/if}} {{#if proxy_url }} proxy_url: {{proxy_url}} {{/if}} - tags: {{#if preserve_original_event}} -- preserve_original_event + - preserve_original_event {{/if}} {{#each tags as |tag i|}} -- {{tag}} + - {{tag}} {{/each}} {{#contains "forwarded" tags}} publisher_pipeline.disable_host: true