Add cloudwatch input into AWS package for log collection

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.10.0"
+  changes:
+    - description: Add cloudwatch input into AWS package for log collection
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2323
 - version: "1.9.0"
   changes:
     - description: Add Route 53 Resolver Logs Datastream

packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs

@@ -0,0 +1,93 @@
+{{#unless log_group_name}}
+{{#unless log_group_name_prefix}}
+{{#if log_group_arn }}
+log_group_arn: {{ log_group_arn }}
+{{/if}}
+{{/unless}}
+{{/unless}}
+
+{{#unless log_group_arn}}
+{{#unless log_group_name}}
+{{#if log_group_name_prefix }}
+log_group_name_prefix: {{ log_group_name_prefix }}
+{{/if}}
+{{/unless}}
+{{/unless}}
+
+{{#unless log_group_arn}}
+{{#unless log_group_name_prefix}}
+{{#if log_group_name }}
+log_group_name: {{ log_group_name }}
+{{/if}}
+{{/unless}}
+{{/unless}}
+
+{{#unless log_group_arn}}
+region_name: {{ region_name }}
+{{/unless}}
+
+{{#unless log_stream_prefix}}
+{{#if log_streams }}
+log_streams: {{ log_streams }}
+{{/if}}
+{{/unless}}
+
+{{#unless log_streams}}
+{{#if log_stream_prefix }}
+log_stream_prefix: {{ log_stream_prefix }}
+{{/if}}
+{{/unless}}
+
+{{#if start_position }}
+start_position: {{ start_position }}
+{{/if}}
+
+{{#if scan_frequency }}
+scan_frequency: {{ scan_frequency }}
+{{/if}}
+
+{{#if api_sleep }}
+api_sleep: {{ api_sleep }}
+{{/if}}
+
+{{#if credential_profile_name}}
+credential_profile_name: {{credential_profile_name}}
+{{/if}}
+{{#if shared_credential_file}}
+shared_credential_file: {{shared_credential_file}}
+{{/if}}
+{{#if api_timeout}}
+api_timeout: {{api_timeout}}
+{{/if}}
+{{#if endpoint}}
+endpoint: {{endpoint}}
+{{/if}}
+{{#if access_key_id}}
+access_key_id: {{access_key_id}}
+{{/if}}
+{{#if secret_access_key}}
+secret_access_key: {{secret_access_key}}
+{{/if}}
+{{#if session_token}}
+session_token: {{session_token}}
+{{/if}}
+{{#if role_arn}}
+role_arn: {{role_arn}}
+{{/if}}
+{{#if proxy_url }}
+proxy_url: {{proxy_url}}
+{{/if}}
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/aws/data_stream/cloudtrail/manifest.yml

@@ -193,3 +193,109 @@ streams:
         type: bool
         multi: false
         default: false
+  - input: aws-cloudwatch
+    template_path: aws-cloudwatch.yml.hbs
+    title: AWS CloudTrail Logs
+    description: Collect AWS CloudTrail logs using cloudwatch input
+    enabled: false
+    vars:
+      - name: log_group_arn
+        type: text
+        title: Log Group ARN
+        multi: false
+        required: false
+        show_user: true
+        description: ARN of the log group to collect logs from.
+      - name: log_group_name
+        type: text
+        title: Log Group Name
+        multi: false
+        required: false
+        show_user: false
+        description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given.
+      - name: log_group_name_prefix
+        type: text
+        title: Log Group Name Prefix
+        multi: false
+        required: false
+        show_user: false
+        description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: region_name
+        type: text
+        title: Region Name
+        multi: false
+        required: false
+        show_user: false
+        description: Region that the specified log group or log group prefix belongs to.
+      - name: log_streams
+        type: text
+        title: Log Streams
+        multi: true
+        required: false
+        show_user: false
+        description: A list of strings of log streams names that Filebeat collect log events from.
+      - name: log_streams_prefix
+        type: text
+        title: Log Stream Prefix
+        multi: false
+        required: false
+        show_user: false
+        description: A string to filter the results to include only log events from log streams that have names starting with this prefix.
+      - name: start_position
+        type: text
+        title: Start Position
+        multi: false
+        required: false
+        default: beginning
+        show_user: true
+        description: Allows user to specify if this input should read log files from the beginning or from the end.
+      - name: scan_frequency
+        type: text
+        title: Scan Frequency
+        multi: false
+        required: false
+        show_user: false
+        default: 1m
+        description: This config parameter sets how often Filebeat checks for new log events from the specified log group.
+      - name: api_timeput
+        type: text
+        title: API Timeout
+        multi: false
+        required: false
+        show_user: false
+        default: 120s
+        description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted.
+      - name: api_sleep
+        type: text
+        title: API Sleep
+        multi: false
+        required: false
+        show_user: false
+        default: 200ms
+        description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account.
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: true
+        show_user: false
+        default:
+          - forwarded
+          - aws-cloudtrail
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false

packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs

@@ -0,0 +1,93 @@
+{{#unless log_group_name}}
+{{#unless log_group_name_prefix}}
+{{#if log_group_arn }}
+log_group_arn: {{ log_group_arn }}
+{{/if}}
+{{/unless}}
+{{/unless}}
+
+{{#unless log_group_arn}}
+{{#unless log_group_name}}
+{{#if log_group_name_prefix }}
+log_group_name_prefix: {{ log_group_name_prefix }}
+{{/if}}
+{{/unless}}
+{{/unless}}
+
+{{#unless log_group_arn}}
+{{#unless log_group_name_prefix}}
+{{#if log_group_name }}
+log_group_name: {{ log_group_name }}
+{{/if}}
+{{/unless}}
+{{/unless}}
+
+{{#unless log_group_arn}}
+region_name: {{ region_name }}
+{{/unless}}
+
+{{#unless log_stream_prefix}}
+{{#if log_streams }}
+log_streams: {{ log_streams }}
+{{/if}}
+{{/unless}}
+
+{{#unless log_streams}}
+{{#if log_stream_prefix }}
+log_stream_prefix: {{ log_stream_prefix }}
+{{/if}}
+{{/unless}}
+
+{{#if start_position }}
+start_position: {{ start_position }}
+{{/if}}
+
+{{#if scan_frequency }}
+scan_frequency: {{ scan_frequency }}
+{{/if}}
+
+{{#if api_sleep }}
+api_sleep: {{ api_sleep }}
+{{/if}}
+
+{{#if credential_profile_name}}
+credential_profile_name: {{credential_profile_name}}
+{{/if}}
+{{#if shared_credential_file}}
+shared_credential_file: {{shared_credential_file}}
+{{/if}}
+{{#if api_timeout}}
+api_timeout: {{api_timeout}}
+{{/if}}
+{{#if endpoint}}
+endpoint: {{endpoint}}
+{{/if}}
+{{#if access_key_id}}
+access_key_id: {{access_key_id}}
+{{/if}}
+{{#if secret_access_key}}
+secret_access_key: {{secret_access_key}}
+{{/if}}
+{{#if session_token}}
+session_token: {{session_token}}
+{{/if}}
+{{#if role_arn}}
+role_arn: {{role_arn}}
+{{/if}}
+{{#if proxy_url }}
+proxy_url: {{proxy_url}}
+{{/if}}
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}