diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 77a5a8c1859..ddf200c99d2 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.10.0" + changes: + - description: Add cloudwatch input into AWS package for log collection + type: enhancement + link: https://github.com/elastic/integrations/pull/2323 - version: "1.9.0" changes: - description: Add Route 53 Resolver Logs Datastream diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..c1576eedae0 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,93 @@ +{{#unless log_group_name}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} +{{/if}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} +{{/if}} +{{/unless}} + +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} +{{/if}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} +{{/if}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml index 5db302b0f85..236e763fabc 100644 --- a/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/packages/aws/data_stream/cloudtrail/manifest.yml @@ -193,3 +193,109 @@ streams: type: bool multi: false default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS CloudTrail Logs + description: Collect AWS CloudTrail logs using cloudwatch input + enabled: false + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-cloudtrail + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..c1576eedae0 --- /dev/null +++ b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,93 @@ +{{#unless log_group_name}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} +{{/if}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} +{{/if}} +{{/unless}} + +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} +{{/if}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} +{{/if}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml index 7608cf40377..db207724a22 100644 --- a/packages/aws/data_stream/cloudwatch_logs/manifest.yml +++ b/packages/aws/data_stream/cloudwatch_logs/manifest.yml @@ -3,7 +3,8 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS CloudWatch logs + title: AWS CloudWatch logs via S3 + enabled: false description: Collect AWS CloudWatch logs using s3 input vars: - name: visibility_timeout @@ -68,3 +69,108 @@ streams: default: 5 required: false show_user: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS CloudWatch logs via CloudWatch + description: Collect AWS CloudWatch logs using cloudwatch input. + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-cloudwatch-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..c1576eedae0 --- /dev/null +++ b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,93 @@ +{{#unless log_group_name}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} +{{/if}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} +{{/if}} +{{/unless}} + +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} +{{/if}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} +{{/if}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml index 3c7e8961cd5..fc4749fa8c0 100644 --- a/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/packages/aws/data_stream/ec2_logs/manifest.yml @@ -3,8 +3,9 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS EC2 logs + title: AWS EC2 Logs via S3 description: Collect AWS EC2 logs using s3 input + enabled: false vars: - name: visibility_timeout type: text @@ -68,3 +69,108 @@ streams: default: 5 required: false show_user: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS EC2 Logs via CloudWatch + description: Collect AWS EC2 logs using cloudwatch input. + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-ec2-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..c1576eedae0 --- /dev/null +++ b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,93 @@ +{{#unless log_group_name}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} +{{/if}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} +{{/if}} +{{/unless}} + +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} +{{/if}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} +{{/if}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml index fdd4f2549e4..22b780f2137 100644 --- a/packages/aws/data_stream/elb_logs/manifest.yml +++ b/packages/aws/data_stream/elb_logs/manifest.yml @@ -3,7 +3,7 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS ELB logs + title: AWS ELB Logs via S3 description: Collect AWS ELB logs using s3 input vars: - name: visibility_timeout @@ -68,3 +68,109 @@ streams: default: 5 required: false show_user: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS ELB Logs via CloudWatch + description: Collect AWS ELB logs using cloudwatch input. + enabled: false + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-elb-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..c1576eedae0 --- /dev/null +++ b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,93 @@ +{{#unless log_group_name}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} +{{/if}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} +{{/if}} +{{/unless}} + +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} +{{/if}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} +{{/if}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml index 605ffc87124..86b60d82fc0 100644 --- a/packages/aws/data_stream/firewall_logs/manifest.yml +++ b/packages/aws/data_stream/firewall_logs/manifest.yml @@ -61,3 +61,109 @@ streams: type: bool multi: false default: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS ELB Logs via CloudWatch + description: Collect AWS ELB logs using cloudwatch input. + enabled: false + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-firewall-logs + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml index b88c6ba4900..8240665215a 100644 --- a/packages/aws/data_stream/s3access/manifest.yml +++ b/packages/aws/data_stream/s3access/manifest.yml @@ -3,7 +3,7 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS s3access logs + title: AWS S3 Access Logs description: Collect AWS s3access logs using s3 input vars: - name: visibility_timeout diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..c1576eedae0 --- /dev/null +++ b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,93 @@ +{{#unless log_group_name}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} +{{/if}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} +{{/if}} +{{/unless}} + +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} +{{/if}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} +{{/if}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml index 3812639fb81..5ecd3064156 100644 --- a/packages/aws/data_stream/vpcflow/manifest.yml +++ b/packages/aws/data_stream/vpcflow/manifest.yml @@ -3,8 +3,9 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS vpcflow logs + title: AWS VPC Flow Logs via S3 description: Collect AWS vpcflow logs using s3 input + enabled: false vars: - name: visibility_timeout type: text @@ -68,3 +69,108 @@ streams: default: 5 required: false show_user: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS VPC Flow Logs via CloudWatch + description: Collect AWS VPC flow logs using cloudwatch input. + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-vpcflow + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs new file mode 100644 index 00000000000..c1576eedae0 --- /dev/null +++ b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs @@ -0,0 +1,93 @@ +{{#unless log_group_name}} +{{#unless log_group_name_prefix}} +{{#if log_group_arn }} +log_group_arn: {{ log_group_arn }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name}} +{{#if log_group_name_prefix }} +log_group_name_prefix: {{ log_group_name_prefix }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +{{#unless log_group_name_prefix}} +{{#if log_group_name }} +log_group_name: {{ log_group_name }} +{{/if}} +{{/unless}} +{{/unless}} + +{{#unless log_group_arn}} +region_name: {{ region_name }} +{{/unless}} + +{{#unless log_stream_prefix}} +{{#if log_streams }} +log_streams: {{ log_streams }} +{{/if}} +{{/unless}} + +{{#unless log_streams}} +{{#if log_stream_prefix }} +log_stream_prefix: {{ log_stream_prefix }} +{{/if}} +{{/unless}} + +{{#if start_position }} +start_position: {{ start_position }} +{{/if}} + +{{#if scan_frequency }} +scan_frequency: {{ scan_frequency }} +{{/if}} + +{{#if api_sleep }} +api_sleep: {{ api_sleep }} +{{/if}} + +{{#if credential_profile_name}} +credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if shared_credential_file}} +shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if api_timeout}} +api_timeout: {{api_timeout}} +{{/if}} +{{#if endpoint}} +endpoint: {{endpoint}} +{{/if}} +{{#if access_key_id}} +access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} +secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} +session_token: {{session_token}} +{{/if}} +{{#if role_arn}} +role_arn: {{role_arn}} +{{/if}} +{{#if proxy_url }} +proxy_url: {{proxy_url}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml index 9abff552b53..bc5bdb5c336 100644 --- a/packages/aws/data_stream/waf/manifest.yml +++ b/packages/aws/data_stream/waf/manifest.yml @@ -3,7 +3,7 @@ type: logs streams: - input: aws-s3 template_path: aws-s3.yml.hbs - title: AWS WAF logs + title: AWS WAF logs via S3 description: Collect AWS WAF logs using s3 input vars: - name: visibility_timeout @@ -68,3 +68,109 @@ streams: default: 5 required: false show_user: false + - input: aws-cloudwatch + template_path: aws-cloudwatch.yml.hbs + title: AWS WAF logs via CloudWatch + description: Collect AWS WAF logs using cloudwatch input + enabled: false + vars: + - name: log_group_arn + type: text + title: Log Group ARN + multi: false + required: false + show_user: true + description: ARN of the log group to collect logs from. + - name: log_group_name + type: text + title: Log Group Name + multi: false + required: false + show_user: false + description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given. + - name: log_group_name_prefix + type: text + title: Log Group Name Prefix + multi: false + required: false + show_user: false + description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + - name: region_name + type: text + title: Region Name + multi: false + required: false + show_user: false + description: Region that the specified log group or log group prefix belongs to. + - name: log_streams + type: text + title: Log Streams + multi: true + required: false + show_user: false + description: A list of strings of log streams names that Filebeat collect log events from. + - name: log_streams_prefix + type: text + title: Log Stream Prefix + multi: false + required: false + show_user: false + description: A string to filter the results to include only log events from log streams that have names starting with this prefix. + - name: start_position + type: text + title: Start Position + multi: false + required: false + default: beginning + show_user: true + description: Allows user to specify if this input should read log files from the beginning or from the end. + - name: scan_frequency + type: text + title: Scan Frequency + multi: false + required: false + show_user: false + default: 1m + description: This config parameter sets how often Filebeat checks for new log events from the specified log group. + - name: api_timeput + type: text + title: API Timeout + multi: false + required: false + show_user: false + default: 120s + description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted. + - name: api_sleep + type: text + title: API Sleep + multi: false + required: false + show_user: false + default: 200ms + description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account. + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - aws-waf + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index fe79810c111..3fbdb411c31 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 1.9.0 +version: 1.10.0 license: basic description: Collect logs and metrics from Amazon Web Services with Elastic Agent. type: integration @@ -104,8 +104,12 @@ policy_templates: - security inputs: - type: aws-s3 - title: Collect logs from Cloudtrail service - description: Collecting Cloudtrail logs using aws-s3 input + title: Collect logs from S3 + description: Collecting logs using aws-s3 input + input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting logs using aws-cloudwatch input input_group: logs - type: httpjson title: Collect logs from third-party REST API (experimental) @@ -129,8 +133,12 @@ policy_templates: - cloudwatch_metrics inputs: - type: aws-s3 + title: Collect logs from S3 + description: Collecting logs using aws-s3 input + input_group: logs + - type: aws-cloudwatch title: Collect logs from CloudWatch - description: Collecting logs from CloudWatch using aws-s3 input + description: Collecting logs using aws-cloudwatch input input_group: logs - type: aws/metrics title: Collect metrics from CloudWatch @@ -193,8 +201,12 @@ policy_templates: - ec2_metrics inputs: - type: aws-s3 - title: Collect logs from EC2 service - description: Collecting EC2 logs using aws-s3 input + title: Collect logs from S3 + description: Collecting logs using aws-s3 input + input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting logs using aws-cloudwatch input input_group: logs - type: aws/metrics title: Collect metrics from EC2 service @@ -220,8 +232,12 @@ policy_templates: - network inputs: - type: aws-s3 - title: Collect logs from ELB service - description: Collecting ELB logs using aws-s3 input + title: Collect logs from S3 + description: Collecting logs from ELB using aws-s3 input + input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting logs from ELB using aws-cloudwatch input input_group: logs - type: aws/metrics title: Collect metrics from ELB service @@ -288,9 +304,13 @@ policy_templates: - firewall_metrics inputs: - type: aws-s3 - title: Collect logs from Network Firewall + title: Collect logs from S3 description: Collecting logs from Network Firewall using aws-s3 input input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting logs from Network Firewall using aws-cloudwatch input + input_group: logs - type: aws/metrics title: Collect metrics from Network Firewall description: Collecting metrics from AWS Network Firewall @@ -351,7 +371,7 @@ policy_templates: - security inputs: - type: aws-s3 - title: Collect S3 access logs + title: Collect logs from S3 description: Collecting S3 access logs using aws-s3 input input_group: logs - type: aws/metrics @@ -476,9 +496,13 @@ policy_templates: - security inputs: - type: aws-s3 - title: Collect VPC Flow logs + title: Collect logs from S3 description: Collecting VPC Flow logs using aws-s3 input input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting VPC Flow logs using aws-cloudwatch input + input_group: logs icons: - src: /img/logo_vpcflow.svg title: AWS VPC logo @@ -511,9 +535,13 @@ policy_templates: - security inputs: - type: aws-s3 - title: Collect WAF logs + title: Collect logs from S3 description: Collecting WAF logs using aws-s3 input input_group: logs + - type: aws-cloudwatch + title: Collect logs from CloudWatch + description: Collecting WAF logs using aws-cloudwatch input + input_group: logs icons: - src: /img/logo_waf.svg title: AWS VPC logo