[Cloud-Security] Adding var-groups to support authentication vars - related to cloud connector feature

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "6.15.0"
+  changes:
+    - description: Add var_groups for credential type selection with Cloud Connector support for agentless deployments.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/18762
 - version: "6.14.0"
   changes:
     - description: Enable agentless deployment for AWS RDS metrics.

packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml

@@ -3,97 +3,119 @@ description: "Pipeline for API Gateway logs in CloudWatch"
 
 processors:
   - set:
+      tag: set_ecs_version
       field: ecs.version
       value: '8.11.0'
   - rename:
+      tag: rename_message
       field: message
       target_field: event.original
       ignore_missing: true
       if: 'ctx.event?.original == null'
       description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.'
   - remove:
+      tag: remove_message
       field: message
       ignore_missing: true
       if: 'ctx.event?.original != null'
       description: 'The `message` field is no longer required if the document has an `event.original` field.'
   - json:
+      tag: json_event_original
       field: event.original
       target_field: aws.apigateway
       ignore_failure: true
   - rename:
+      tag: rename_aws_apigateway_requestid
       field: aws.apigateway.requestId
       target_field: aws.apigateway.request_id
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_responselength
       field: aws.apigateway.responseLength
       target_field: aws.apigateway.response_length
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_requesttime
       field: aws.apigateway.requestTime
       target_field: aws.apigateway.request_time
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_httpmethod
       field: aws.apigateway.httpMethod
       target_field: aws.apigateway.http_method
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_routekey
       field: aws.apigateway.routeKey
       target_field: aws.apigateway.route_key
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_ip
       field: aws.apigateway.ip
       target_field: aws.apigateway.ip_address
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_resourcepath
       field: aws.apigateway.resourcePath
       target_field: aws.apigateway.resource_path
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_connectionid
       field: aws.apigateway.connectionId
       target_field: aws.apigateway.connection_id
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_eventtype
       field: aws.apigateway.eventType
       target_field: aws.apigateway.event_type
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_apiid
       field: aws.apigateway.apiId
       target_field: aws.apigateway.api_id
       ignore_missing: true
   - rename:
+      tag: rename_aws_apigateway_domainname
       field: aws.apigateway.domainName
       target_field: aws.apigateway.domain_name
       ignore_missing: true
   - grok:
+      tag: grok_aws_apigateway_ip_address
       field: aws.apigateway.ip_address
       patterns:
         - '%{IPORHOST:aws.apigateway.ip_address}'
       ignore_failure: true
   - convert:
+      tag: convert_aws_apigateway_ip_address
       field: aws.apigateway.ip_address
       type: ip
       ignore_missing: true
   - convert:
+      tag: convert_aws_apigateway_response_length
       field: aws.apigateway.response_length
       type: long
       ignore_missing: true
   - convert:
+      tag: convert_aws_apigateway_status
       field: aws.apigateway.status
       type: long
       ignore_missing: true
   - date:
+      tag: date_aws_apigateway_request_time
       field: aws.apigateway.request_time
       target_field: "aws.apigateway.request_time"
       formats:
         - dd/MMM/yyyy:H:m:s Z
       ignore_failure: true
 on_failure:
   - set:
+      tag: set_event_kind
       field: event.kind
       value: pipeline_error
   - append:
+      tag: append_error_message
       field: error.message
       value: >-
         Processor '{{{ _ingest.on_failure_processor_type }}}'
         {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
-        {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
+        {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'

packages/aws/data_stream/awshealth/elasticsearch/ingest_pipeline/default.yml

@@ -2,11 +2,13 @@
 description: Pipeline for AWS Health metrics
 processors:
 - script:
+    tag: script
     if: "ctx.aws != null && ctx.aws.awshealth != null && ctx.aws.awshealth.end_time == '0001-01-01T00:00:00.000Z'"
     "lang": "painless"
     "source": "ctx.aws.awshealth.end_time = null"
 
 - script:
+    tag: script_1
     lang: painless
     source: |-
         boolean drop(Object o) {
@@ -26,11 +28,13 @@ processors:
 
 on_failure:
 - set:
+    tag: set_event_kind
     field: event.kind
     value: pipeline_error
 - append:
+    tag: append_error_message
     field: error.message
     value: >-
       Processor '{{{ _ingest.on_failure_processor_type }}}'
       {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
-      {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
+      {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'

packages/aws/data_stream/billing/elasticsearch/ingest_pipeline/default.yml

@@ -2,24 +2,29 @@
 description: "Pipeline for AWS Billing"
 processors:
   - dot_expander:
+      tag: dot_expander_all
       field: "*"
       ignore_failure: true
   - set:
+      tag: set_cloud_account_name
       field: cloud.account.name
       copy_from: cloud.account.id
       override: false
       ignore_empty_value: true
   - fingerprint:
+      tag: fingerprint
       fields: ["aws.billing.group_by"]
       target_field: 'aws.billing.group_by.fingerprint'
       ignore_missing: true
 on_failure:
   - set:
+      tag: set_event_kind
       field: event.kind
       value: pipeline_error
   - append:
+      tag: append_error_message
       field: error.message
       value: >-
         Processor '{{{ _ingest.on_failure_processor_type }}}'
         {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
-        {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
+        {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}'