Add cloudwatch input into AWS package for log collection

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.8.0"
+  changes:
+    - description: Add cloudwatch input into AWS package for log collection
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2323
 - version: "1.7.0"
   changes:
     - description: Add integration for AWS Network Firewall

packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs

@@ -0,0 +1,76 @@
+{{#unless log_group_name}}
+{{#if log_group_arn}}
+log_group_arn: {{log_group_arn}}
+{{/if}}
+{{/unless}}
+
+{{#unless log_group_arn}}
+{{#if log_group_name}}
+log_group_name: {{log_group_name}}
+{{/if}}
+{{#if log_group_name_prefix}}
+log_group_name_prefix: {{log_group_name_prefix}}
+{{/if}}
+{{#if region_name}}
+region_name: {{region_name}}
+{{/if}}
+{{/unless}}
+
+{{#if log_streams}}
+log_streams: {{log_streams}}
+{{/if}}
+{{#if log_stream_prefix}}
+log_stream_prefix: {{log_stream_prefix}}
+{{/if}}
+{{#if start_position}}
+start_position: {{start_position}}
+{{/if}}
+{{#if scan_frequency}}
+scan_frequency: {{scan_frequency}}
+{{/if}}
+{{#if api_timeout}}
+api_timeout: {{api_timeout}}
+{{/if}}
+{{#if api_sleep}}
+api_sleep: {{api_sleep}}
+{{/if}}
+
+{{#if access_key_id}}
+access_key_id: {{access_key_id}}
+{{/if}}
+{{#if secret_access_key}}
+secret_access_key: {{secret_access_key}}
+{{/if}}
+{{#if session_token}}
+session_token: {{session_token}}
+{{/if}}
+{{#if role_arn}}
+role_arn: {{role_arn}}
+{{/if}}
+{{#if credential_profile_name}}
+credential_profile_name: {{credential_profile_name}}
+{{/if}}
+{{#if shared_credential_file}}
+shared_credential_file: {{shared_credential_file}}
+{{/if}}
+{{#if endpoint}}
+endpoint: {{endpoint}}
+{{/if}}
+{{#if proxy_url }}
+proxy_url: {{proxy_url}}
+{{/if}}
+
+tags:
+{{#if preserve_original_event}}
+- preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/aws/data_stream/cloudtrail/manifest.yml

@@ -3,8 +3,9 @@ type: logs
 streams:
   - input: aws-s3
     template_path: aws-s3.yml.hbs
-    title: AWS CloudTrail Logs
-    description: Collect AWS CloudTrail logs using s3 input
+    title: AWS CloudTrail Logs via S3
+    description: Collect logs using s3 input
+    enabled: false
     vars:
       - name: visibility_timeout
         type: text
@@ -193,3 +194,107 @@ streams:
         type: bool
         multi: false
         default: false
+  - input: aws-cloudwatch
+    template_path: aws-cloudwatch.yml.hbs
+    title: AWS CloudTrail Logs via CloudWatch
+    description: Collect logs using cloudwatch input
+    vars:
+      - name: log_group_arn
+        type: text
+        title: Log Group ARN
+        multi: false
+        required: false
+        show_user: true
+        description: ARN of the log group to collect logs from.
+      - name: log_group_name
+        type: text
+        title: Log Group Name
+        multi: false
+        required: false
+        show_user: false
+        description: Name of the log group to collect logs from. `region_name` is required when `log_group_name` is given.
+      - name: log_group_name_prefix
+        type: text
+        title: Log Group Name Prefix
+        multi: false
+        required: false
+        show_user: false
+        description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: region_name
+        type: text
+        title: Region Name
+        multi: false
+        required: false
+        show_user: false
+        description: Region that the specified log group or log group prefix belongs to.
+      - name: log_streams
+        type: text
+        title: Log Streams
+        multi: true
+        required: false
+        show_user: false
+        description: A list of strings of log streams names that Filebeat collect log events from.
+      - name: log_streams_prefix
+        type: text
+        title: Log Stream Prefix
+        multi: false
+        required: false
+        show_user: false
+        description: A string to filter the results to include only log events from log streams that have names starting with this prefix.
+      - name: start_position
+        type: text
+        title: Start Position
+        multi: false
+        required: false
+        default: beginning
+        show_user: true
+        description: Allows user to specify if this input should read log files from the beginning or from the end.
+      - name: scan_frequency
+        type: text
+        title: Scan Frequency
+        multi: false
+        required: false
+        show_user: false
+        default: 1m
+        description: This config parameter sets how often Filebeat checks for new log events from the specified log group.
+      - name: api_timeput
+        type: text
+        title: API Timeout
+        multi: false
+        required: false
+        show_user: false
+        default: 120s
+        description: The maximum duration of AWS API can take. If it exceeds the timeout, AWS API will be interrupted.
+      - name: api_sleep
+        type: text
+        title: API Sleep
+        multi: false
+        required: false
+        show_user: false
+        default: 200ms
+        description: This is used to sleep between AWS FilterLogEvents API calls inside the same collection period. `FilterLogEvents` API has a quota of 5 transactions per second (TPS)/account/Region. This value should only be adjusted when there are multiple Filebeats or multiple Filebeat inputs collecting logs from the same region and AWS account.
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+        default:
+          - forwarded
+          - aws-cloudtrail
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`
+        type: bool
+        multi: false
+        default: false

packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs

@@ -0,0 +1,76 @@
+{{#unless log_group_name}}
+{{#if log_group_arn}}
+log_group_arn: {{log_group_arn}}
+{{/if}}
+{{/unless}}
+
+{{#unless log_group_arn}}
+{{#if log_group_name}}
+log_group_name: {{log_group_name}}
+{{/if}}
+{{#if log_group_name_prefix}}
+log_group_name_prefix: {{log_group_name_prefix}}
+{{/if}}
+{{#if region_name}}
+region_name: {{region_name}}
+{{/if}}
+{{/unless}}
+
+{{#if log_streams}}
+log_streams: {{log_streams}}
+{{/if}}
+{{#if log_stream_prefix}}
+log_stream_prefix: {{log_stream_prefix}}
+{{/if}}
+{{#if start_position}}
+start_position: {{start_position}}
+{{/if}}
+{{#if scan_frequency}}
+scan_frequency: {{scan_frequency}}
+{{/if}}
+{{#if api_timeout}}
+api_timeout: {{api_timeout}}
+{{/if}}
+{{#if api_sleep}}
+api_sleep: {{api_sleep}}
+{{/if}}
+
+{{#if access_key_id}}
+access_key_id: {{access_key_id}}
+{{/if}}
+{{#if secret_access_key}}
+secret_access_key: {{secret_access_key}}
+{{/if}}
+{{#if session_token}}
+session_token: {{session_token}}
+{{/if}}
+{{#if role_arn}}
+role_arn: {{role_arn}}
+{{/if}}
+{{#if credential_profile_name}}
+credential_profile_name: {{credential_profile_name}}
+{{/if}}
+{{#if shared_credential_file}}
+shared_credential_file: {{shared_credential_file}}
+{{/if}}
+{{#if endpoint}}
+endpoint: {{endpoint}}
+{{/if}}
+{{#if proxy_url }}
+proxy_url: {{proxy_url}}
+{{/if}}
+
+tags:
+{{#if preserve_original_event}}
+- preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+- {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}