diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index b42983cc32b..df81d69461a 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "6.15.0" + changes: + - description: Add var_groups for credential type selection with Cloud Connector support for agentless deployments. + type: enhancement + link: https://github.com/elastic/integrations/pull/18762 - version: "6.14.0" changes: - description: Enable agentless deployment for AWS RDS metrics. diff --git a/packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml index c01edeac459..ece3c74316f 100644 --- a/packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,85 +3,105 @@ description: "Pipeline for API Gateway logs in CloudWatch" processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original field: event.original target_field: aws.apigateway ignore_failure: true - rename: + tag: rename_aws_apigateway_requestid field: aws.apigateway.requestId target_field: aws.apigateway.request_id ignore_missing: true - rename: + tag: rename_aws_apigateway_responselength field: aws.apigateway.responseLength target_field: aws.apigateway.response_length ignore_missing: true - rename: + tag: rename_aws_apigateway_requesttime field: aws.apigateway.requestTime target_field: aws.apigateway.request_time ignore_missing: true - rename: + tag: rename_aws_apigateway_httpmethod field: aws.apigateway.httpMethod target_field: aws.apigateway.http_method ignore_missing: true - rename: + tag: rename_aws_apigateway_routekey field: aws.apigateway.routeKey target_field: aws.apigateway.route_key ignore_missing: true - rename: + tag: rename_aws_apigateway_ip field: aws.apigateway.ip target_field: aws.apigateway.ip_address ignore_missing: true - rename: + tag: rename_aws_apigateway_resourcepath field: aws.apigateway.resourcePath target_field: aws.apigateway.resource_path ignore_missing: true - rename: + tag: rename_aws_apigateway_connectionid field: aws.apigateway.connectionId target_field: aws.apigateway.connection_id ignore_missing: true - rename: + tag: rename_aws_apigateway_eventtype field: aws.apigateway.eventType target_field: aws.apigateway.event_type ignore_missing: true - rename: + tag: rename_aws_apigateway_apiid field: aws.apigateway.apiId target_field: aws.apigateway.api_id ignore_missing: true - rename: + tag: rename_aws_apigateway_domainname field: aws.apigateway.domainName target_field: aws.apigateway.domain_name ignore_missing: true - grok: + tag: grok_aws_apigateway_ip_address field: aws.apigateway.ip_address patterns: - '%{IPORHOST:aws.apigateway.ip_address}' ignore_failure: true - convert: + tag: convert_aws_apigateway_ip_address field: aws.apigateway.ip_address type: ip ignore_missing: true - convert: + tag: convert_aws_apigateway_response_length field: aws.apigateway.response_length type: long ignore_missing: true - convert: + tag: convert_aws_apigateway_status field: aws.apigateway.status type: long ignore_missing: true - date: + tag: date_aws_apigateway_request_time field: aws.apigateway.request_time target_field: "aws.apigateway.request_time" formats: @@ -89,11 +109,13 @@ processors: ignore_failure: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/awshealth/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/awshealth/elasticsearch/ingest_pipeline/default.yml index 0e137279bc7..c5861d4d307 100644 --- a/packages/aws/data_stream/awshealth/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/awshealth/elasticsearch/ingest_pipeline/default.yml @@ -2,11 +2,13 @@ description: Pipeline for AWS Health metrics processors: - script: + tag: script if: "ctx.aws != null && ctx.aws.awshealth != null && ctx.aws.awshealth.end_time == '0001-01-01T00:00:00.000Z'" "lang": "painless" "source": "ctx.aws.awshealth.end_time = null" - script: + tag: script_1 lang: painless source: |- boolean drop(Object o) { @@ -26,11 +28,13 @@ processors: on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/billing/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/billing/elasticsearch/ingest_pipeline/default.yml index 75cc75192f5..492bba6c09b 100644 --- a/packages/aws/data_stream/billing/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/billing/elasticsearch/ingest_pipeline/default.yml @@ -2,24 +2,29 @@ description: "Pipeline for AWS Billing" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true - fingerprint: + tag: fingerprint fields: ["aws.billing.group_by"] target_field: 'aws.billing.group_by.fingerprint' ignore_missing: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml index e545fdffdbe..5ec3c507d68 100644 --- a/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,35 +3,44 @@ description: "Pipeline for CloudFront standard access logs" processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - set: + tag: set_event_kind field: event.kind value: event - set: + tag: set_event_category field: event.category value: ["web"] - append: + tag: append_event_type field: event.type value: ["access"] - set: + tag: set_cloud_provider field: cloud.provider value: aws - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: ctx.event?.original == null description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - drop: + tag: drop if: ctx.event?.original?.startsWith('#') description: "Drop if logline contains header(s), which startswith `#`" - csv: + tag: csv_event_original description: Parse cloudfront csv row field: event.original separator: "\t" @@ -75,6 +84,7 @@ processors: # date # time - script: + tag: script description: timestamp composed of date and time. Script processor performs better than equivalent set processor. lang: painless source: >- @@ -82,6 +92,7 @@ processors: if: ctx._tmp?.date != null && ctx._tmp?.time != null - date: + tag: date_tmp_timestamp field: _tmp.timestamp target_field: '@timestamp' ignore_failure: true @@ -91,117 +102,140 @@ processors: if: ctx._tmp?.timestamp != null # x-edge-location - rename: + tag: rename_tmp_x_edge_location field: _tmp.x_edge_location target_field: aws.cloudfront.edge_location ignore_missing: true # sc-bytes - convert: + tag: convert_tmp_sc_bytes field: _tmp.sc_bytes target_field: http.response.bytes type: long ignore_missing: true # c-ip - rename: + tag: rename_tmp_c_ip field: _tmp.c_ip target_field: source.address ignore_missing: true - convert: + tag: convert_source_address field: source.address target_field: source.ip type: ip ignore_missing: true ignore_failure: true - append: + tag: append_related_ip field: related.ip value: "{{{ source.ip }}}" if: ctx.source?.ip != null # cs-method - rename: + tag: rename_tmp_cs_method field: _tmp.cs_method target_field: http.request.method ignore_missing: true # cs-host - rename: + tag: rename_tmp_cs_host field: _tmp.cs_host target_field: aws.cloudfront.domain ignore_missing: true - append: + tag: append_related_hosts field: related.hosts value: "{{{ aws.cloudfront.domain }}}" allow_duplicates: false if: ctx.aws?.cloudfront?.domain != null # cs-uri-stem - rename: + tag: rename_tmp_cs_uri_stem field: _tmp.cs_uri_stem target_field: url.path ignore_missing: true # cs-status - convert: + tag: convert_tmp_cs_status field: _tmp.cs_status target_field: http.response.status_code type: long ignore_missing: true # cs-referer - rename: + tag: rename_tmp_cs_referer field: _tmp.cs_referer target_field: http.request.referrer ignore_missing: true if: ctx._tmp?.cs_referer != null && ctx._tmp.cs_referer != '-' # cs(User-Agent) - urldecode: + tag: urldecode_tmp_cs_user_agent field: _tmp.cs_user_agent ignore_missing: true - user_agent: + tag: user_agent_tmp_cs_user_agent field: _tmp.cs_user_agent target_field: user_agent ignore_missing: true # cs-uri-query - rename: + tag: rename_tmp_cs_uri_query field: _tmp.cs_uri_query target_field: url.query if: ctx._tmp?.cs_uri_query != null && ctx._tmp.cs_uri_query != '-' # cs(Cookie) - rename: + tag: rename_tmp_cs_cookie field: _tmp.cs_cookie target_field: aws.cloudfront.cookies if: ctx._tmp?.cs_cookie != null && ctx._tmp.cs_cookie != '-' # x-edge-result-type - rename: + tag: rename_tmp_x_edge_result_type field: _tmp.x_edge_result_type target_field: aws.cloudfront.edge_result_type ignore_missing: true # x-edge-request-id - rename: + tag: rename_tmp_x_edge_request_id field: _tmp.x_edge_request_id target_field: http.request.id ignore_missing: true # x-host-header - rename: + tag: rename_tmp_x_host_header field: _tmp.x_host_header target_field: destination.address ignore_missing: true - set: + tag: set_destination_domain field: destination.domain copy_from: destination.address ignore_empty_value: true - append: + tag: append_related_hosts_1 field: related.hosts value: "{{{ destination.domain }}}" allow_duplicates: false if: ctx.destination?.domain != null # cs-protocol - rename: + tag: rename_tmp_cs_protocol field: _tmp.cs_protocol target_field: network.protocol ignore_missing: true # cs-bytes - convert: + tag: convert_tmp_cs_bytes field: _tmp.cs_bytes target_field: http.request.bytes type: long ignore_missing: true # time-taken - script: + tag: script_1 lang: painless if: ctx._tmp?.time_taken != null params: @@ -210,11 +244,13 @@ processors: ctx.event.duration = (Long)(Float.parseFloat(ctx._tmp.time_taken) * params.S_TO_NS); # x-forwarded-for - split: + tag: split_tmp_x_forwarded_for field: _tmp.x_forwarded_for separator: "," target_field: _tmp.split_x_forwarded_for if: ctx._tmp?.x_forwarded_for != null && ctx._tmp.x_forwarded_for != '-' - script: + tag: script_2 lang: painless description: trim leading and trailing whitespace from the split IPs if: ctx._tmp?.split_x_forwarded_for != null @@ -223,6 +259,7 @@ processors: ctx._tmp.split_x_forwarded_for[i] = ctx._tmp.split_x_forwarded_for[i].trim(); } - foreach: + tag: foreach_tmp_split_x_forwarded_for field: _tmp.split_x_forwarded_for if: ctx._tmp.split_x_forwarded_for instanceof List processor: @@ -231,10 +268,12 @@ processors: tag: pipeline_process_ip ignore_missing_pipeline: true - append: + tag: append_error_message field: error.message value: "Invalid IP addresses: {{_tmp.invalid_ips}}" if: ctx._tmp.invalid_ips != null - script: + tag: script_3 lang: painless description: Handle 'localhost' edge case, currently not handled via grok if: ctx._tmp?.split_x_forwarded_for != null && ctx._tmp.split_x_forwarded_for != '-' @@ -252,6 +291,7 @@ processors: } } - foreach: + tag: foreach_network_forwarded_ip field: network.forwarded_ip processor: append: @@ -260,6 +300,7 @@ processors: ignore_missing: true # ssl-protocol - grok: + tag: grok_tmp_ssl_protocol field: _tmp.ssl_protocol patterns: - '(-|%{TLS:tls.version_protocol}v%{NUMBER:tls.version})' @@ -267,64 +308,76 @@ processors: TLS: '(TLS|SSL)' ignore_missing: true - lowercase: + tag: lowercase_tls_version_protocol field: tls.version_protocol ignore_missing: true # ssl-cipher - rename: + tag: rename_tmp_ssl_cipher field: _tmp.ssl_cipher target_field: tls.cipher ignore_missing: true if: ctx._tmp?.ssl_cipher != null && ctx._tmp?.ssl_cipher != '-' # x-edge-response-result-type - rename: + tag: rename_tmp_x_edge_response_result_type field: _tmp.x_edge_response_result_type target_field: aws.cloudfront.edge_response_result_type ignore_missing: true # x-edge-response-result-type - rename: + tag: rename_tmp_x_edge_response_result_type_1 field: _tmp.x_edge_response_result_type target_field: aws.cloudfront.edge_response_result_type ignore_missing: true # cs-protocol-version - dissect: + tag: dissect_tmp_cs_protocol_version field: _tmp.cs_protocol_version pattern: "%{}/%{http.version}" ignore_missing: true ignore_failure: true # fle-status - rename: + tag: rename_tmp_fle_status field: _tmp.fle_status target_field: aws.cloudfront.fle_status if: ctx._tmp?.fle_status != null && ctx._tmp.fle_status != '-' # fle-encrypted-fields - rename: + tag: rename_tmp_fle_encrypted_fields field: _tmp.fle_encrypted_fields target_field: aws.cloudfront.fle_encrypted_fields if: ctx._tmp?.encrypted_fields != null && ctx._tmp.encrypted_fields != '-' # c-port - convert: + tag: convert_tmp_c_port field: _tmp.c_port target_field: source.port type: long if: ctx._tmp?.c_port != null && ctx._tmp.c_port != '-' # time-to-first-byte - convert: + tag: convert_tmp_time_to_first_byte field: _tmp.time_to_first_byte target_field: aws.cloudfront.time_to_first_byte type: float if: ctx._tmp?.time_to_first_byte != null && ctx._tmp.time_to_first_byte != '-' # x-edge-detailed-result-type - rename: + tag: rename_tmp_x_edge_detailed_result_type field: _tmp.x_edge_detailed_result_type target_field: aws.cloudfront.edge_detailed_result_type if: ctx._tmp?.x_edge_detailed_result_type != null && ctx._tmp.x_edge_detailed_result_type != '-' # sc-content-type - rename: + tag: rename_tmp_sc_content_type field: _tmp.sc_content_type target_field: http.response.mime_type ignore_missing: true # sc-content-len - convert: + tag: convert_tmp_sc_content_len field: _tmp.sc_content_len target_field: http.response.body.bytes type: long @@ -332,6 +385,7 @@ processors: if: ctx._tmp?.sc_content_len != null && ctx._tmp.sc_content_len != '-' # sc-range-start - convert: + tag: convert_tmp_sc_range_start field: _tmp.sc_range_start target_field: aws.cloudfront.range_start type: long @@ -339,6 +393,7 @@ processors: if: ctx._tmp?.sc_range_start != null && ctx._tmp.sc_range_start != '-' # sc-range-end - convert: + tag: convert_tmp_sc_range_end field: _tmp.sc_range_end target_field: aws.cloudfront.range_end type: long @@ -349,6 +404,7 @@ processors: ### Additional fields ############################################################################## # url - script: + tag: script_4 lang: painless description: This script builds the `url.full` field out of the available `url.*` parts. source: | @@ -369,34 +425,41 @@ processors: ctx._tmp.url_full = full } - uri_parts: + tag: uri_parts_tmp_url_full field: _tmp.url_full target_field: url keep_original: false ignore_missing: true - rename: + tag: rename_tmp_url_full field: _tmp.url_full target_field: url.full ignore_missing: true - registered_domain: + tag: registered_domain_url_domain field: url.domain target_field: url ignore_missing: true # Network type - set: + tag: set_network_type field: network.type value: ipv4 if: ctx.source?.ip != null && ctx.source.ip.contains('.') - set: + tag: set_network_type_1 field: network.type value: ipv6 if: ctx.source?.ip != null && ctx.source.ip.contains(':') # IP Geolocation Lookup - geoip: + tag: geoip_source_ip field: source.ip target_field: source.geo ignore_missing: true # IP Autonomous System (AS) Lookup - geoip: + tag: geoip_source_ip_1 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -405,42 +468,51 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true # event.id - set: + tag: set_event_id field: event.id copy_from: http.request.id ignore_empty_value: true # event.outcome - set: + tag: set_event_outcome field: event.outcome value: failure if: ctx.http?.response?.status_code >= 400 - set: + tag: set_event_outcome_1 field: event.outcome value: success if: ctx.http?.response?.status_code < 400 && ctx.http?.response?.status_code > 000 - set: + tag: set_event_outcome_2 field: event.outcome value: failure if: ctx.http?.response?.status_code == 000 # cleanup - remove: + tag: remove_tmp field: _tmp ignore_missing: true on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message_1 field: error.message value: >- Processor '{{ _ingest.on_failure_processor_type }}' {{#_ingest.on_failure_processor_tag}}with tag '{{ _ingest.on_failure_processor_tag }}' - {{/_ingest.on_failure_processor_tag}}failed with message '{{ _ingest.on_failure_message }}' + {{/_ingest.on_failure_processor_tag}}in pipeline '{{ _ingest.pipeline }}' failed with message '{{ _ingest.on_failure_message }}' diff --git a/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/pipeline_process_ip.yml b/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/pipeline_process_ip.yml index ef02239cd9a..204c10a04e2 100644 --- a/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/pipeline_process_ip.yml +++ b/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/pipeline_process_ip.yml @@ -16,18 +16,35 @@ processors: IPV6NOCOMPRESS: '([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}' on_failure: - set: + tag: set_tmp_invalid_ip field: _tmp.invalid_ip value: "{{_ingest._value}}" - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - append: + tag: append_network_forwarded_ip field: network.forwarded_ip value: "{{_tmp.valid_ip}}" if: ctx._tmp.valid_ip != null && ctx._tmp.valid_ip != "" allow_duplicates: false - append: + tag: append_tmp_invalid_ips field: _tmp.invalid_ips value: "{{_tmp.invalid_ip}}" if: ctx._tmp.invalid_ip != null && ctx._tmp.invalid_ip != "" allow_duplicates: false + +on_failure: + - set: + tag: set_event_kind_pipeline_error + field: event.kind + value: pipeline_error + - append: + tag: append_pipeline_error_message + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml index a9cec976b2e..72eab824eaf 100644 --- a/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml @@ -2,26 +2,32 @@ description: Pipeline for AWS CloudTrail Logs processors: - rename: + tag: rename_message field: message target_field: event.original if: ctx.event?.original == null description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - set: + tag: set_event_created if: ctx['@timestamp'] != null field: event.created copy_from: '@timestamp' - json: + tag: json_event_original field: event.original target_field: json - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - date: + tag: date_json_eventtime field: json.eventTime target_field: "@timestamp" ignore_failure: true @@ -130,10 +136,12 @@ processors: 'AssumedRole': true on_failure: - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false - set: + tag: set_error_message description: Add error reason field: error.message value: | @@ -1011,6 +1019,7 @@ processors: field("related.entity").set(enrichCtx.related); - rename: + tag: rename_json_eventversion field: json.eventVersion target_field: aws.cloudtrail.event_version ignore_failure: true @@ -1030,28 +1039,34 @@ processors: ignore_missing: true tag: rename_user_identity_principal_id - rename: + tag: rename_json_useridentity_arn field: json.userIdentity.arn target_field: aws.cloudtrail.user_identity.arn ignore_failure: true - rename: + tag: rename_json_useridentity_accountid field: json.userIdentity.accountId target_field: cloud.account.id ignore_failure: true - rename: + tag: rename_json_useridentity_accesskeyid field: json.userIdentity.accessKeyId target_field: aws.cloudtrail.user_identity.access_key_id ignore_failure: true - rename: + tag: rename_json_useridentity_sessioncontext_attributes_mfaauthenticated field: json.userIdentity.sessionContext.attributes.mfaAuthenticated target_field: aws.cloudtrail.user_identity.session_context.mfa_authenticated ignore_failure: true - date: + tag: date_json_useridentity_sessioncontext_attributes_creationdate field: json.userIdentity.sessionContext.attributes.creationDate target_field: aws.cloudtrail.user_identity.session_context.creation_date ignore_failure: true formats: - ISO8601 - rename: + tag: rename_json_useridentity_sessioncontext_sessionissuer_type field: json.userIdentity.sessionContext.sessionIssuer.type target_field: aws.cloudtrail.user_identity.session_context.session_issuer.type ignore_failure: true @@ -1064,6 +1079,7 @@ processors: if: ctx.aws?.cloudtrail?.user_identity?.type == 'AssumedRole' || ctx.aws?.cloudtrail?.user_identity?.type == 'FederatedUser' tag: rename_user_name_assumed_role - rename: + tag: rename_json_useridentity_sessioncontext_sessionissuer_principalid field: json.userIdentity.sessionContext.sessionIssuer.principalId target_field: aws.cloudtrail.user_identity.session_context.session_issuer.principal_id ignore_missing: true @@ -1082,32 +1098,39 @@ processors: if: ctx.aws?.cloudtrail?.user_identity?.type == 'IdentityCenterUser' && ctx.user?.id == null tag: rename_user_id_identity_center_user - rename: + tag: rename_json_useridentity_sessioncontext_sessionissuer_arn field: json.userIdentity.sessionContext.sessionIssuer.arn target_field: aws.cloudtrail.user_identity.session_context.session_issuer.arn ignore_failure: true - rename: + tag: rename_json_useridentity_sessioncontext_sessionissuer_accountid field: json.userIdentity.sessionContext.sessionIssuer.accountId target_field: aws.cloudtrail.user_identity.session_context.session_issuer.account_id ignore_failure: true - rename: + tag: rename_json_sessioncredentialfromconsole field: json.sessionCredentialFromConsole target_field: aws.cloudtrail.session_credential_from_console ignore_missing: true ignore_failure: true - rename: + tag: rename_json_useridentity_invokedby field: json.userIdentity.invokedBy target_field: aws.cloudtrail.user_identity.invoked_by ignore_failure: true - rename: + tag: rename_json_eventsource field: json.eventSource target_field: event.provider ignore_failure: true - set: + tag: set_event_action field: event.action value: '{{{json.eventName}}}' ignore_failure: true ignore_empty_value: true - rename: + tag: rename_json_eventcategory field: json.eventCategory target_field: aws.cloudtrail.event_category ignore_failure: true @@ -1117,24 +1140,29 @@ processors: target_field: user.name if: ctx.event?.action == 'UserAuthentication' && ctx.user?.name == null && ctx.json?.additionalEventData?.UserName != null - set: + tag: set_cloud_region field: cloud.region copy_from: json.awsRegion ignore_empty_value: true - rename: + tag: rename_json_sourceipaddress field: json.sourceIPAddress target_field: source.address ignore_failure: true - grok: + tag: grok_source_address field: source.address ignore_failure: true patterns: - ^%{IP:source.ip}$ - geoip: + tag: geoip_source_ip field: source.ip target_field: source.geo ignore_failure: true ignore_missing: true - geoip: + tag: geoip_source_ip_1 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -1143,26 +1171,32 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - user_agent: + tag: user_agent_json_useragent field: json.userAgent target_field: user_agent on_failure: - rename: + tag: rename_json_useragent field: json.userAgent target_field: user_agent.original ignore_failure: true - rename: + tag: rename_json_errorcode field: json.errorCode target_field: aws.cloudtrail.error_code ignore_failure: true - rename: + tag: rename_json_errormessage field: json.errorMessage target_field: aws.cloudtrail.error_message ignore_failure: true @@ -1217,22 +1251,27 @@ processors: } ignore_failure: true - rename: + tag: rename_json_requestid field: json.requestID target_field: aws.cloudtrail.request_id ignore_failure: true - rename: + tag: rename_json_eventid field: json.eventID target_field: event.id ignore_failure: true - rename: + tag: rename_json_eventtype field: json.eventType target_field: aws.cloudtrail.event_type ignore_failure: true - rename: + tag: rename_json_apiversion field: json.apiVersion target_field: aws.cloudtrail.api_version ignore_failure: true - script: + tag: script lang: painless source: | if (ctx.json?.managementEvent != null) { @@ -1240,10 +1279,12 @@ processors: } ignore_failure: true - rename: + tag: rename_json_readonly field: json.readOnly target_field: aws.cloudtrail.read_only ignore_failure: true - script: + tag: script_1 description: 'Drops duplicates where the same combination of ARN, accountId and type is found multiple times in the resources list.' lang: painless source: > @@ -1270,42 +1311,51 @@ processors: ctx.json.resources = new ArrayList(uniqueResources.values()); } - rename: + tag: rename_json_resources field: json.resources target_field: aws.cloudtrail.resources ignore_failure: true - rename: + tag: rename_json_recipientaccountid field: json.recipientAccountId target_field: aws.cloudtrail.recipient_account_id ignore_failure: true - set: + tag: set_cloud_account_id field: cloud.account.id copy_from: aws.cloudtrail.recipient_account_id override: false ignore_empty_value: true - rename: + tag: rename_json_sharedeventid field: json.sharedEventId target_field: aws.cloudtrail.shared_event_id ignore_failure: true - rename: + tag: rename_json_vpcendpointid field: json.vpcEndpointId target_field: aws.cloudtrail.vpc_endpoint_id ignore_failure: true - append: + tag: append_related_user field: related.user value: '{{json.requestParameters.userName}}' allow_duplicates: false if: ctx.json?.requestParameters?.userName != null - append: + tag: append_related_user_1 field: related.user value: '{{json.requestParameters.newUserName}}' allow_duplicates: false if: ctx.json?.requestParameters?.newUserName != null - append: + tag: append_related_user_2 field: related.user value: '{{json.responseElements.user.userId}}' allow_duplicates: false if: ctx.json?.responseElements?.user?.userId != null - script: + tag: script_2 lang: painless ignore_failure: true source: >- @@ -1856,129 +1906,158 @@ processors: hm.forEach((k, v) -> ctx.event[k] = v); - rename: + tag: rename_json_awsaccountid field: json.awsAccountId target_field: cloud.account.id ignore_failure: true - rename: + tag: rename_json_digests3object field: json.digestS3Object target_field: file.path ignore_failure: true - rename: + tag: rename_json_previousdigestsignature field: json.previousDigestSignature target_field: file.hash.sha256 if: >- ctx.json?.previousDigestHashAlgorithm != null && ctx.json?.previousDigestHashAlgorithm == 'SHA-256' - append: + tag: append_related_hash field: related.hash value: '{{{file.hash.sha256}}}' if: ctx.file?.hash?.sha256 != null - rename: + tag: rename_json_logfiles field: json.logFiles target_field: aws.cloudtrail.digest.log_files ignore_failure: true - date: + tag: date_json_digeststarttime field: json.digestStartTime target_field: aws.cloudtrail.digest.start_time ignore_failure: true formats: - ISO8601 - date: + tag: date_json_digestendtime field: json.digestEndTime target_field: "@timestamp" ignore_failure: true formats: - ISO8601 - date: + tag: date_json_digestendtime_1 field: json.digestEndTime target_field: aws.cloudtrail.digest.end_time ignore_failure: true formats: - ISO8601 - rename: + tag: rename_json_digests3bucket field: json.digestS3Bucket target_field: aws.cloudtrail.digest.s3_bucket ignore_failure: true - date: + tag: date_json_newesteventtime field: json.newestEventTime target_field: aws.cloudtrail.digest.newest_event_time ignore_failure: true formats: - ISO8601 - date: + tag: date_json_oldesteventtime field: json.oldestEventTime target_field: aws.cloudtrail.digest.oldest_event_time ignore_failure: true formats: - ISO8601 - rename: + tag: rename_json_previousdigests3bucket field: json.previousDigestS3Bucket target_field: aws.cloudtrail.digest.previous_s3_bucket ignore_failure: true - rename: + tag: rename_json_previousdigesthashalgorithm field: json.previousDigestHashAlgorithm target_field: aws.cloudtrail.digest.previous_hash_algorithm ignore_failure: true - rename: + tag: rename_json_publickeyfingerprint field: json.publicKeyFingerprint target_field: aws.cloudtrail.digest.public_key_fingerprint ignore_failure: true - rename: + tag: rename_json_digestsignaturealgorithm field: json.digestSignatureAlgorithm target_field: aws.cloudtrail.digest.signature_algorithm ignore_failure: true - set: + tag: set_group_id field: group.id copy_from: json.responseElements.group.groupId ignore_empty_value: true - set: + tag: set_user_target_id field: user.target.id copy_from: json.responseElements.user.userId ignore_empty_value: true - set: + tag: set_user_changes_name field: user.changes.name copy_from: json.requestParameters.newUserName ignore_empty_value: true - set: + tag: set_group_name field: group.name copy_from: json.requestParameters.groupName ignore_empty_value: true - set: + tag: set_user_target_name field: user.target.name copy_from: json.requestParameters.userName ignore_empty_value: true - remove: + tag: remove field: - aws.cloudtrail.digest - json.insightDetails if: '!ctx._conf.keep_flattened_duplicates' ignore_missing: true - rename: + tag: rename_aws_cloudtrail_digest field: aws.cloudtrail.digest target_field: aws.cloudtrail.flattened.digest ignore_missing: true - rename: + tag: rename_json_insightdetails field: json.insightDetails target_field: aws.cloudtrail.flattened.insight_details ignore_missing: true - remove: + tag: remove_json_tlsdetails field: json.tlsDetails if: ctx.json?.tlsDetails?.tlsVersion == 'tlsVersion' - dissect: + tag: dissect_json_tlsdetails_tlsversion field: json.tlsDetails.tlsVersion pattern: "%{tls.version_protocol}v%{tls.version}" ignore_missing: true on_failure: - rename: + tag: rename_json_tlsdetails_tlsversion field: json.tlsDetails.tlsVersion target_field: tls.version - lowercase: + tag: lowercase_tls_version_protocol field: tls.version_protocol ignore_missing: true - rename: + tag: rename_json_tlsdetails_ciphersuite field: json.tlsDetails.cipherSuite target_field: tls.cipher ignore_missing: true - rename: + tag: rename_json_tlsdetails_clientprovidedhostheader field: json.tlsDetails.clientProvidedHostHeader target_field: tls.client.server_name ignore_missing: true @@ -1991,12 +2070,14 @@ processors: # Don't dissect the user email into @ # as the parts are meaningless on their own - append: + tag: append_related_user_3 field: related.user value: '{{{user.id}}}' if: ctx.user?.id != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_user_4 field: related.user value: '{{{user.name}}}' if: ctx.user?.name != null @@ -2004,6 +2085,7 @@ processors: ignore_failure: true - append: + tag: append_related_user_5 field: related.user value: '{{{user.changes.name}}}' if: ctx.user?.changes?.name != null @@ -2052,6 +2134,7 @@ processors: } ctx.aws.cloudtrail.flattened = flattened; - remove: + tag: remove_1 field: - aws.cloudtrail.response_elements - aws.cloudtrail.request_parameters @@ -2060,6 +2143,7 @@ processors: if: ctx._conf?.retain != null && ctx._conf.retain != '' && ctx._conf.retain != 'all' && ctx._conf.retain != 'keyword' && ctx._conf.retain != 'minimal' - remove: + tag: remove_2 field: - json - _conf @@ -2093,15 +2177,18 @@ processors: on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' - remove: + tag: remove_3 field: - json - _conf diff --git a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml index 7c9c84bc23b..e88e69fa3e0 100644 --- a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,26 +3,32 @@ description: "Pipeline for logs ingested from CloudWatch" processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - set: + tag: set_event_original field: event.original copy_from: message ignore_empty_value: true if: 'ctx.event?.original == null' - set: + tag: set_cloud_provider field: cloud.provider value: aws - set: + tag: set_event_kind field: event.kind value: event on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/cloudwatch_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudwatch_metrics/elasticsearch/ingest_pipeline/default.yml index 3e4b1c904ee..664f5a0489b 100644 --- a/packages/aws/data_stream/cloudwatch_metrics/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudwatch_metrics/elasticsearch/ingest_pipeline/default.yml @@ -2,16 +2,19 @@ description: "Pipeline for AWS CloudWatch metrics" processors: - fingerprint: + tag: fingerprint fields: ["aws.dimensions"] target_field: 'aws.dimensions_fingerprint' ignore_missing: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/config/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/config/elasticsearch/ingest_pipeline/default.yml index df22ad199bf..291a8e7a62d 100644 --- a/packages/aws/data_stream/config/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/config/elasticsearch/ingest_pipeline/default.yml @@ -39,6 +39,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -118,6 +119,7 @@ processors: target_field: aws.config.rule_info.config_rule_state ignore_missing: true - append: + tag: append_related_user field: related.user value: '{{{json.ConfigRuleInfo.CreatedBy}}}' allow_duplicates: false @@ -143,6 +145,7 @@ processors: copy_from: aws.config.rule_info.description ignore_empty_value: true - foreach: + tag: foreach_json_configruleinfo_evaluationmodes field: json.ConfigRuleInfo.EvaluationModes if: ctx.json?.ConfigRuleInfo?.EvaluationModes instanceof List processor: @@ -163,6 +166,7 @@ processors: target_field: aws.config.rule_info.input_parameters on_failure: - append: + tag: append_error_message_1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -193,6 +197,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -211,6 +216,7 @@ processors: target_field: aws.config.rule_info.source.owner ignore_missing: true - foreach: + tag: foreach_json_configruleinfo_source_sourcedetails field: json.ConfigRuleInfo.Source.SourceDetails if: ctx.json?.ConfigRuleInfo?.Source?.SourceDetails instanceof List processor: @@ -220,6 +226,7 @@ processors: target_field: _ingest._value.event_source ignore_missing: true - foreach: + tag: foreach_json_configruleinfo_source_sourcedetails_1 field: json.ConfigRuleInfo.Source.SourceDetails if: ctx.json?.ConfigRuleInfo?.Source?.SourceDetails instanceof List processor: @@ -229,6 +236,7 @@ processors: target_field: _ingest._value.maximum_execution_frequency ignore_missing: true - foreach: + tag: foreach_json_configruleinfo_source_sourcedetails_2 field: json.ConfigRuleInfo.Source.SourceDetails if: ctx.json?.ConfigRuleInfo?.Source?.SourceDetails instanceof List processor: @@ -257,6 +265,7 @@ processors: if: ctx.json?.ConfigRuleInvokedTime != null && ctx.json.ConfigRuleInvokedTime != '' on_failure: - append: + tag: append_error_message_3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -279,6 +288,7 @@ processors: if: ctx.json?.EvaluationResultIdentifier?.OrderingTimestamp != null && ctx.json.EvaluationResultIdentifier.OrderingTimestamp != '' on_failure: - append: + tag: append_error_message_4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -306,6 +316,7 @@ processors: if: ctx.json?.ResultRecordedTime != null && ctx.json.ResultRecordedTime != '' on_failure: - append: + tag: append_error_message_5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -426,19 +437,22 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_6 field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: field: event.kind tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags_1 field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/aws/data_stream/dynamodb/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/dynamodb/elasticsearch/ingest_pipeline/default.yml index 930555aecfb..26eb7ce3239 100644 --- a/packages/aws/data_stream/dynamodb/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/dynamodb/elasticsearch/ingest_pipeline/default.yml @@ -3,20 +3,24 @@ description: "Pipeline for DynamoDB metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml index edeac7d4c91..553f268e843 100644 --- a/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,40 +3,48 @@ description: "Pipeline for EC2 logs in CloudWatch" processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - grok: + tag: grok_event_original field: event.original patterns: - '%{TIMESTAMP_ISO8601:_tmp.timestamp} %{SYSLOGTIMESTAMP:_tmp.syslog_timestamp} %{IPORHOST:aws.ec2.ip_address} %{DATA:process.name}(?:\\[%{POSINT:process.pid}\\])?: %{GREEDYDATA:message}' - date: + tag: date_tmp_timestamp field: _tmp.timestamp target_field: '@timestamp' ignore_failure: true formats: - ISO8601 - remove: + tag: remove field: - _tmp ignore_missing: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/ec2_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/ec2_metrics/elasticsearch/ingest_pipeline/default.yml index 52be576741b..db6ae28cd36 100644 --- a/packages/aws/data_stream/ec2_metrics/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/ec2_metrics/elasticsearch/ingest_pipeline/default.yml @@ -3,14 +3,17 @@ description: "Pipeline for EC2 metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true - script: + tag: script lang: painless description: This script converts aws.ec2.metrics.CPUUtilization.avg from percentage to decimal. source: | @@ -18,47 +21,56 @@ processors: ctx.aws.ec2.metrics.CPUUtilization.avg = ctx.aws.ec2.metrics.CPUUtilization.avg / 100; } - rename: + tag: rename_aws_ec2_metrics_cpuutilization_avg field: aws.ec2.metrics.CPUUtilization.avg target_field: host.cpu.usage ignore_missing: true ignore_failure: true - rename: + tag: rename_aws_ec2_metrics_networkin_sum field: aws.ec2.metrics.NetworkIn.sum target_field: host.network.ingress.bytes ignore_missing: true ignore_failure: true - rename: + tag: rename_aws_ec2_metrics_networkout_sum field: aws.ec2.metrics.NetworkOut.sum target_field: host.network.egress.bytes ignore_missing: true ignore_failure: true - rename: + tag: rename_aws_ec2_metrics_networkpacketsin_sum field: aws.ec2.metrics.NetworkPacketsIn.sum target_field: host.network.ingress.packets ignore_missing: true ignore_failure: true - rename: + tag: rename_aws_ec2_metrics_networkpacketsout_sum field: aws.ec2.metrics.NetworkPacketsOut.sum target_field: host.network.egress.packets ignore_missing: true ignore_failure: true - rename: + tag: rename_aws_ec2_metrics_diskreadbytes_sum field: aws.ec2.metrics.DiskReadBytes.sum target_field: host.disk.read.bytes ignore_missing: true ignore_failure: true - rename: + tag: rename_aws_ec2_metrics_diskwritebytes_sum field: aws.ec2.metrics.DiskWriteBytes.sum target_field: host.disk.write.bytes ignore_missing: true ignore_failure: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml index a36dc4c75c6..a87900b910b 100644 --- a/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,20 +3,24 @@ description: "Pipeline for ELB logs" processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - grok: + tag: grok_event_original field: event.original # Classic ELB patterns documented in https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html # ELB v2 Application load balancers https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html @@ -122,86 +126,106 @@ processors: ELBV2TYPE: '%{WORD:aws.elb.type}' ELBV2LOGVERSION: '%{NOTSPACE}' # Could be used to support different log versions, only 1.0 exists now - set: + tag: set_event_kind field: event.kind value: event - set: + tag: set_cloud_provider field: cloud.provider value: aws - set: + tag: set_aws_elb_protocol if: ctx.http != null field: aws.elb.protocol value: http - uri_parts: + tag: uri_parts_tmp_uri_orig if: 'ctx?._tmp?.uri_orig != null' field: _tmp.uri_orig ignore_failure: true - user_agent: + tag: user_agent_tmp_user_agent if: 'ctx?._tmp?.user_agent != null' field: _tmp.user_agent ignore_missing: true - set: + tag: set_event_category if: ctx.http != null field: event.category value: [web] - set: + tag: set_aws_elb_protocol_1 field: aws.elb.protocol value: tcp if: ctx.http == null - set: + tag: set_event_category_1 field: event.category value: [network] if: ctx.http == null - set: + tag: set_event_outcome field: event.outcome value: success if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400' - set: + tag: set_event_outcome_1 field: event.outcome value: failure if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400' - set: + tag: set_trace_id field: trace.id value: '{{aws.elb.trace_id}}' if: ctx?.aws?.elb?.trace_id != null - split: + tag: split_tmp_actions_executed field: _tmp.actions_executed target_field: aws.elb.action_executed separator: ',' ignore_missing: true - split: + tag: split_tmp_target_port field: _tmp.target_port target_field: aws.elb.target_port separator: ' ' ignore_missing: true - split: + tag: split_tmp_target_status_code field: _tmp.target_status_code target_field: aws.elb.target_status_code separator: ' ' ignore_missing: true - date: + tag: date_tmp_timestamp field: _tmp.timestamp formats: - ISO8601 - set: + tag: set_event_end field: event.end value: '{{ @timestamp }}' - convert: + tag: convert_source_address field: source.address target_field: source.ip type: ip ignore_failure: true - convert: + tag: convert_source_port field: source.port type: long ignore_failure: true - geoip: + tag: geoip_source_ip field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_1 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -210,18 +234,22 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - set: + tag: set_tls_cipher field: tls.cipher value: '{{aws.elb.ssl_cipher}}' if: ctx.aws?.elb?.ssl_cipher != null - script: + tag: script lang: painless if: ctx.aws?.elb?.ssl_protocol != null source: >- @@ -236,41 +264,49 @@ processors: } ctx.tls.version_protocol = parts[0].toLowerCase(); - remove: + tag: remove field: - _tmp ignore_missing: true - date: + tag: date_aws_elb_tls_connection_creation_time_str field: aws.elb.tls_connection_creation_time_str target_field: aws.elb.tls_connection_creation_time formats: ["ISO8601"] "if": "ctx.aws?.elb?.tls_connection_creation_time_str != null && ctx.aws?.elb?.tls_connection_creation_time_str != '-' && ctx.aws?.elb?.tls_connection_creation_time_str != ''" - remove: + tag: remove_aws_elb_tls_connection_creation_time_str field: aws.elb.tls_connection_creation_time_str ignore_missing: true - date: + tag: date_aws_elb_leaf_client_cert_not_after_str field: aws.elb.leaf_client_cert_not_after_str target_field: aws.elb.leaf_client_cert_not_after formats: ["ISO8601"] "if": "ctx.aws?.elb?.leaf_client_cert_not_after_str != null && ctx.aws?.elb?.leaf_client_cert_not_after_str != '-' && ctx.aws?.elb?.leaf_client_cert_not_after_str != ''" - date: + tag: date_aws_elb_leaf_client_cert_not_before_str field: aws.elb.leaf_client_cert_not_before_str target_field: aws.elb.leaf_client_cert_not_before formats: ["ISO8601"] "if": "ctx.aws?.elb?.leaf_client_cert_not_before_str != null && ctx.aws?.elb?.leaf_client_cert_not_before_str != '-' && ctx.aws?.elb?.leaf_client_cert_not_before_str != ''" - remove: + tag: remove_aws_elb_leaf_client_cert_not_after_str field: ["aws.elb.leaf_client_cert_not_after_str", "aws.elb.leaf_client_cert_not_before_str"] ignore_missing: true on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/emr_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/emr_logs/elasticsearch/ingest_pipeline/default.yml index 00e6111b1d6..1670885561e 100644 --- a/packages/aws/data_stream/emr_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/emr_logs/elasticsearch/ingest_pipeline/default.yml @@ -2,20 +2,24 @@ description: "Pipeline for EMR logs" processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - grok: + tag: grok_event_original field: event.original pattern_definitions: GREEDYMULTILINE: "(.|\\n)*" @@ -23,6 +27,7 @@ processors: - '%{TIMESTAMP_ISO8601:_tmp.timestamp}%{SPACE}%{LOGLEVEL:log.level}%{SPACE}%{DATA:process.name}(?:\\[%{GREEDYDATA:process.entrypoint}\\])?:%{SPACE}%{GREEDYDATA:message}%{SPACE}%{GREEDYMULTILINE:process.message}' ignore_missing: true - date: + tag: date_tmp_timestamp field: _tmp.timestamp target_field: '@timestamp' ignore_failure: true @@ -30,10 +35,12 @@ processors: - ISO8601 - yyyy-MM-dd HH:mm:ss,SSS - remove: + tag: remove field: - _tmp ignore_missing: true - script: + tag: script description: Drops null/empty values recursively lang: painless ignore_failure: true @@ -53,11 +60,13 @@ processors: drop(ctx); on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml index 683dd403cd8..4757e16ff6b 100644 --- a/packages/aws/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,33 +3,40 @@ description: "Pipeline for AWS Network Firewall logs" processors: # General data - set: + tag: set_ecs_version field: ecs.version value: 8.11.0 - rename: + tag: rename_message field: message target_field: event.original if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original field: event.original allow_duplicate_keys: true target_field: json - date: + tag: date_json_event_timestamp field: json.event.timestamp target_field: "@timestamp" formats: - ISO8601 ignore_failure: true - rename: + tag: rename_json_availability_zone field: json.availability_zone target_field: cloud.availability_zone ignore_missing: true - grok: + tag: grok_cloud_availability_zone field: cloud.availability_zone ignore_missing: true ignore_failure: true @@ -38,65 +45,80 @@ processors: pattern_definitions: LETTER: '[a-z]+' - rename: + tag: rename_json_firewall_name field: json.firewall_name target_field: observer.name ignore_missing: true - set: + tag: set_observer_type field: observer.type value: firewall - set: + tag: set_observer_vendor field: observer.vendor value: AWS - set: + tag: set_observer_product field: observer.product value: "Network Firewall" # Event metadata - append: + tag: append_event_category field: event.category value: network allow_duplicates: false - append: + tag: append_event_type field: event.type value: connection allow_duplicates: false - set: + tag: set_json_event_event_type field: json.event.event_type value: event if: ctx.json?.event?.event_type == "netflow" - set: + tag: set_event_kind field: event.kind value: "{{json.event.event_type}}" if: ctx.json?.event?.event_type != null - set: + tag: set_json_event_alert_action field: json.event.alert.action value: denied if: ctx.json?.event?.alert?.action == "blocked" - append: + tag: append_event_type_1 field: event.type value: "{{json.event.alert.action}}" if: ctx.json?.event?.alert?.action != null # Source IP/port/geo - convert: + tag: convert_json_event_src_ip field: json.event.src_ip target_field: source.address type: ip ignore_missing: true - set: + tag: set_source_ip field: source.ip copy_from: source.address if: ctx?.source?.address != null - convert: + tag: convert_json_event_src_port field: json.event.src_port type: integer target_field: source.port if: ctx?.json?.event?.src_port != null - geoip: + tag: geoip_source_ip field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_source_ip_1 ignore_missing: true database_file: GeoLite2-ASN.mmdb field: source.ip @@ -105,42 +127,51 @@ processors: - asn - organization_name - rename: + tag: rename_source_as_asn field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - set: + tag: set_network_type field: network.type value: 'ipv4' if: 'ctx.network?.type == null && ctx.source?.ip != null && ctx.source.ip.contains(".")' - set: + tag: set_network_type_1 field: network.type value: 'ipv6' if: 'ctx.network?.type == null && ctx.source?.ip != null && ctx.source.ip.contains(":")' # Destination IP/port/geo - convert: + tag: convert_json_event_dest_ip field: json.event.dest_ip target_field: destination.address type: ip ignore_missing: true - set: + tag: set_destination_ip field: destination.ip copy_from: destination.address if: ctx?.destination?.address != null - convert: + tag: convert_json_event_dest_port field: json.event.dest_port type: integer target_field: destination.port if: ctx?.json?.event?.dest_port != null - geoip: + tag: geoip_destination_ip field: destination.ip target_field: destination.geo ignore_missing: true - geoip: + tag: geoip_destination_ip_1 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -149,151 +180,185 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_destination_as_asn field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true # Transport protocol - rename: + tag: rename_json_event_proto field: json.event.proto target_field: network.transport ignore_missing: true - lowercase: + tag: lowercase_network_transport field: network.transport ignore_missing: true # Alert and metadata - convert: + tag: convert_json_event_alert_category field: json.event.alert.category target_field: message type: string ignore_missing: true - set: + tag: set_rule_category field: rule.category value: "{{json.event.alert.category}}" ignore_empty_value: true - set: + tag: set_rule_id field: rule.id value: "{{json.event.alert.signature_id}}" ignore_empty_value: true - set: + tag: set_rule_name field: rule.name value: "{{json.event.alert.signature}}" ignore_empty_value: true - set: + tag: set_rule_name_1 field: rule.name value: rule.id if: ctx?.rule?.name == null && ctx?.rule?.id != null - rename: + tag: rename_json_event_alert_rev_id field: json.event.alert.rev_id target_field: rule.version ignore_missing: true - rename: + tag: rename_json_event_alert_severity field: json.event.alert.severity target_field: event.severity ignore_missing: true - rename: + tag: rename_json_event_app_proto field: json.event.app_proto target_field: network.protocol ignore_missing: true - set: + tag: set_network_protocol field: network.protocol value: "unknown" if: ctx?.network?.protocol == null || ctx?.network?.protocol == "failed" # HTTP - rename: + tag: rename_json_event_http_hostname field: json.event.http.hostname target_field: destination.domain ignore_missing: true - uri_parts: + tag: uri_parts_json_event_http_url field: json.event.http.url if: ctx?.json?.event?.http?.url != null - rename: + tag: rename_json_event_http_http_method field: json.event.http.http_method target_field: http.request.method ignore_missing: true - user_agent: + tag: user_agent_json_event_http_http_user_agent field: json.event.http.http_user_agent ignore_missing: true - dissect: + tag: dissect_json_event_http_protocol field: json.event.http.protocol pattern: "HTTP/%{http.version}" ignore_missing: true # TLS - rename: + tag: rename_json_event_tls_sni field: json.event.tls.sni target_field: tls.client.server_name ignore_missing: true - set: + tag: set_destination_domain field: destination.domain copy_from: tls.client.server_name if: ctx?.tls?.client?.server_name != null - dissect: + tag: dissect_json_event_tls_version field: json.event.tls.version pattern: "%{tls.version_protocol} %{tls.version}" ignore_missing: true if: ctx?.json?.event?.tls?.version != "UNDETERMINED" - lowercase: + tag: lowercase_tls_version_protocol field: tls.version_protocol ignore_missing: true - rename: + tag: rename_json_event_tls_ja3s_hash field: json.event.tls.ja3s.hash target_field: tls.server.ja3s ignore_missing: true - rename: + tag: rename_json_event_tls_ja3_hash field: json.event.tls.ja3.hash target_field: tls.server.ja3 ignore_missing: true - rename: + tag: rename_json_event_tls_certificate field: json.event.tls.certificate target_field: tls.server.certificate ignore_missing: true - rename: + tag: rename_tls_server_certificate_chain field: tls.server.certificate_chain target_field: json.event.tls.chain ignore_missing: true - rename: + tag: rename_tls_server_x509_serial_number field: tls.server.x509.serial_number target_field: json.event.tls.serial ignore_missing: true - gsub: + tag: gsub_tls_server_x509_serial_number field: tls.server.x509.serial_number pattern: ":" replacement: "" ignore_missing: true - date: + tag: date_json_event_tls_notafter field: json.event.tls.notafter target_field: tls.server.not_after formats: - ISO8601 if: ctx.json?.event?.tls?.notafter != null - date: + tag: date_json_event_tls_notbefore field: json.event.tls.notbefore target_field: tls.server.not_before formats: - ISO8601 if: ctx.json?.event?.tls?.notbefore != null - rename: + tag: rename_tls_server_not_after field: tls.server.not_after target_field: tls.server.x509.not_after ignore_missing: true - rename: + tag: rename_tls_server_not_before field: tls.server.not_before target_field: tls.server.x509.not_before ignore_missing: true # TCP - rename: + tag: rename_json_event_tcp_tcp_flags field: json.event.tcp.tcp_flags target_field: aws.firewall.tcp_flags ignore_missing: true - script: + tag: script lang: painless ignore_failure: true if: "ctx?.aws?.firewall?.tcp_flags != null" @@ -326,30 +391,36 @@ processors: # Flow - rename: + tag: rename_json_event_netflow field: json.event.netflow target_field: aws.firewall.flow ignore_missing: true - rename: + tag: rename_json_event_flow_id field: json.event.flow_id target_field: aws.firewall.flow.id ignore_missing: true - convert: + tag: convert_aws_firewall_flow_id field: aws.firewall.flow.id type: string ignore_missing: true # Related IPs - append: + tag: append_related_hosts field: related.hosts value: "{{url.domain}}" if: ctx.url?.domain != null && ctx.url?.domain != "" allow_duplicates: false - append: + tag: append_related_ip if: ctx?.source?.ip != null field: related.ip value: "{{source.ip}}" allow_duplicates: false - append: + tag: append_related_ip_1 if: ctx?.destination?.ip != null field: related.ip value: "{{destination.ip}}" @@ -357,11 +428,13 @@ processors: # Community ID - community_id: + tag: community_id ignore_missing: true ignore_failure: true # Remove other fields - script: + tag: script_1 lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | @@ -386,15 +459,18 @@ processors: } handleMap(ctx); - remove: + tag: remove_json field: json ignore_missing: true on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-agentless-cloud-connector.expected b/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-agentless-cloud-connector.expected new file mode 100644 index 00000000000..2474bbd3fbf --- /dev/null +++ b/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-agentless-cloud-connector.expected @@ -0,0 +1,86 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: aws + name: test-httpjson-agentless-cloud-connector-aws + streams: + - auth.aws: + external_id: ${SECRET_0} + role_arn: arn:aws:iam::123456789012:role/ElasticGuardDutyReadOnly + use_cloud_connectors: true + chain: + - step: + replace: $.nextToken + request.method: POST + request.ssl: null + request.timeout: 30s + request.transforms: + - set: + target: body.findingIds + value: '[[toJSON .parent_last_response.body.findingIds]]' + value_type: json + - set: + target: body.sortCriteria + value: '{"attributeName":"updatedAt","orderBy":"ASC"}' + value_type: json + request.url: https://guardduty.us-east-1.amazonaws.com/detector/12abc34d567e8fa901bc2d34e567f890/findings/get + response.split: + target: body.findings + config_version: 2 + cursor: + last_execution_datetime: + ignore_empty_value: true + value: '[[$f := (index .last_response.body "findings")]][[if $f]][[if (ne (len $f) 50)]][[.last_event.updatedAt]][[end]][[end]]' + data_stream: + dataset: aws.guardduty + interval: 5m + publisher_pipeline.disable_host: true + request.method: POST + request.ssl: null + request.timeout: 30s + request.transforms: + - set: + target: body.maxResults + value: 50 + value_type: int + - set: + target: body.sortCriteria + value: '{"attributeName":"updatedAt","orderBy":"ASC"}' + value_type: json + - set: + default: '[[((now (parseDuration "-48h"))).UnixMilli]]' + target: body.findingCriteria.criterion.updatedAt.greaterThan + value: '[[((parseDate .cursor.last_execution_datetime)).UnixMilli]]' + - set: + target: body.findingCriteria.criterion.updatedAt.lessThan + value: '[[((now)).UnixMilli]]' + request.url: https://guardduty.us-east-1.amazonaws.com/detector/12abc34d567e8fa901bc2d34e567f890/findings + response.pagination: + - set: + do_not_log_failure: true + fail_on_template_error: true + target: body.nextToken + value: '[[if (ne .last_response.body.nextToken "")]][[.last_response.body.nextToken]][[end]]' + tags: + - forwarded + - aws-guardduty + type: httpjson + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-aws.guardduty-ep + privileges: + - auto_configure + - create_doc +secret_references: + - {} diff --git a/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-agentless-cloud-connector.yml b/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-agentless-cloud-connector.yml new file mode 100644 index 00000000000..18aa8232d34 --- /dev/null +++ b/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-agentless-cloud-connector.yml @@ -0,0 +1,15 @@ +vars: + role_arn: arn:aws:iam::123456789012:role/ElasticGuardDutyReadOnly + external_id: guardduty-external-id + supports_cloud_connectors: true + default_region: us-east-1 +data_stream: + vars: + interval: 5m + initial_interval: 48h + detector_id: 12abc34d567e8fa901bc2d34e567f890 + aws_region: us-east-1 + tld: amazonaws.com + http_client_timeout: 30s + preserve_original_event: false + preserve_duplicate_custom_fields: false diff --git a/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-legacy-credentials.expected b/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-legacy-credentials.expected new file mode 100644 index 00000000000..36c9f757987 --- /dev/null +++ b/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-legacy-credentials.expected @@ -0,0 +1,103 @@ +inputs: + - data_stream: + namespace: ep + meta: + package: + name: aws + name: test-httpjson-legacy-credentials-aws + streams: + - auth.aws: + access_key_id: ${SECRET_0} + secret_access_key: ${SECRET_1} + chain: + - step: + replace: $.nextToken + request.method: POST + request.proxy_url: https://user:P%40ssword%23@192.0.2.10:8080 + request.ssl: + enabled: true + verification_mode: none + request.timeout: 30s + request.transforms: + - set: + target: body.findingIds + value: '[[toJSON .parent_last_response.body.findingIds]]' + value_type: json + - set: + target: body.sortCriteria + value: '{"attributeName":"updatedAt","orderBy":"ASC"}' + value_type: json + request.url: https://guardduty.us-east-1.amazonaws.com/detector/12abc34d567e8fa901bc2d34e567f890/findings/get + response.split: + target: body.findings + config_version: 2 + cursor: + last_execution_datetime: + ignore_empty_value: true + value: '[[$f := (index .last_response.body "findings")]][[if $f]][[if (ne (len $f) 50)]][[.last_event.updatedAt]][[end]][[end]]' + data_stream: + dataset: aws.guardduty + interval: 5m + processors: + - add_fields: + fields: + env: test + name: guardduty + target: project + publisher_pipeline.disable_host: true + request.method: POST + request.proxy_url: https://user:P%40ssword%23@192.0.2.10:8080 + request.ssl: + enabled: true + verification_mode: none + request.timeout: 30s + request.tracer.filename: ../../logs/httpjson/http-request-trace-*.ndjson + request.tracer.maxbackups: 5 + request.transforms: + - set: + target: body.maxResults + value: 50 + value_type: int + - set: + target: body.sortCriteria + value: '{"attributeName":"updatedAt","orderBy":"ASC"}' + value_type: json + - set: + default: '[[((now (parseDuration "-48h"))).UnixMilli]]' + target: body.findingCriteria.criterion.updatedAt.greaterThan + value: '[[((parseDate .cursor.last_execution_datetime)).UnixMilli]]' + - set: + target: body.findingCriteria.criterion.updatedAt.lessThan + value: '[[((now)).UnixMilli]]' + request.url: https://guardduty.us-east-1.amazonaws.com/detector/12abc34d567e8fa901bc2d34e567f890/findings + response.pagination: + - set: + do_not_log_failure: true + fail_on_template_error: true + target: body.nextToken + value: '[[if (ne .last_response.body.nextToken "")]][[.last_response.body.nextToken]][[end]]' + tags: + - preserve_original_event + - preserve_duplicate_custom_fields + - forwarded + - aws-guardduty + - test-policy + type: httpjson + use_output: default +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-aws.guardduty-ep + privileges: + - auto_configure + - create_doc +secret_references: + - {} + - {} diff --git a/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-legacy-credentials.yml b/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-legacy-credentials.yml new file mode 100644 index 00000000000..05098a60a0d --- /dev/null +++ b/packages/aws/data_stream/guardduty/_dev/test/policy/test-httpjson-legacy-credentials.yml @@ -0,0 +1,29 @@ +vars: + access_key_id: FAKE_AWS_ACCESS_KEY_ID_FOR_TESTS_ONLY + secret_access_key: FAKE_AWS_SECRET_ACCESS_KEY_FOR_TESTS_ONLY + default_region: us-east-1 +data_stream: + vars: + enable_request_tracer: true + interval: 5m + initial_interval: 48h + detector_id: 12abc34d567e8fa901bc2d34e567f890 + aws_region: us-east-1 + tld: amazonaws.com + http_client_timeout: 30s + proxy_url: https://user:P%40ssword%23@192.0.2.10:8080 + ssl: | + enabled: true + verification_mode: none + preserve_original_event: true + preserve_duplicate_custom_fields: true + tags: + - forwarded + - aws-guardduty + - test-policy + processors: | + - add_fields: + target: project + fields: + name: guardduty + env: test diff --git a/packages/aws/data_stream/guardduty/agent/stream/httpjson.yml.hbs b/packages/aws/data_stream/guardduty/agent/stream/httpjson.yml.hbs index 85e65511a6c..1bd2debd8d4 100644 --- a/packages/aws/data_stream/guardduty/agent/stream/httpjson.yml.hbs +++ b/packages/aws/data_stream/guardduty/agent/stream/httpjson.yml.hbs @@ -15,10 +15,8 @@ request.proxy_url: {{proxy_url}} {{#if ssl}} request.ssl: {{ssl}} {{/if}} + request.transforms: - - set: - target: header.X-Amz-Date - value: '[[formatDate (now) "20060102T150405Z"]]' - set: target: body.maxResults value: 50 @@ -34,20 +32,12 @@ request.transforms: - set: target: body.findingCriteria.criterion.updatedAt.lessThan value: '[[((now)).UnixMilli]]' - - set: - target: header.Authorization - value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/guardduty/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "guardduty")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/guardduty/aws4_request") (hash "sha256" "POST\n" "/detector/{{detector_id}}/findings\n" "\n" "host:guardduty.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]' response.pagination: - set: target: body.nextToken value: '[[if (ne .last_response.body.nextToken "")]][[.last_response.body.nextToken]][[end]]' fail_on_template_error: true do_not_log_failure: true - - delete: - target: header.Authorization - - set: - target: header.Authorization - value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/guardduty/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "guardduty")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/guardduty/aws4_request") (hash "sha256" "POST\n" "/detector/{{detector_id}}/findings\n" "\n" "host:guardduty.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]' chain: - step: request.url: https://guardduty.{{aws_region}}.{{tld}}/detector/{{detector_id}}/findings/get @@ -63,9 +53,6 @@ chain: {{/if}} request.method: POST request.transforms: - - set: - target: header.X-Amz-Date - value: '[[formatDate (now) "20060102T150405Z"]]' - set: target: body.findingIds value: '[[toJSON .parent_last_response.body.findingIds]]' @@ -74,9 +61,6 @@ chain: target: body.sortCriteria value: '{"attributeName":"updatedAt","orderBy":"ASC"}' value_type: json - - set: - target: header.Authorization - value: '[[$now := (now)]][[(sprintf "AWS4-HMAC-SHA256 Credential={{access_key_id}}/%s/{{aws_region}}/guardduty/aws4_request, SignedHeaders=host;x-amz-date, Signature=%s" (formatDate ($now) "20060102") (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" (hexDecode (hmac "sha256" "AWS4{{secret_access_key}}" (formatDate ($now) "20060102"))) "{{aws_region}}")) "guardduty")) "aws4_request")) "AWS4-HMAC-SHA256\n" (formatDate ($now) "20060102T150405Z") "\n" (sprintf "%s/%s\n" (formatDate ($now) "20060102") "{{aws_region}}/guardduty/aws4_request") (hash "sha256" "POST\n" "/detector/{{detector_id}}/findings/get\n" "\n" "host:guardduty.{{aws_region}}.{{tld}}\n" (sprintf "x-amz-date:%s\n\n" (formatDate ($now) "20060102T150405Z")) "host;x-amz-date\n" (hash "sha256" (sprintf `%s` .body)))))]]' response.split: target: body.findings cursor: @@ -100,3 +84,34 @@ publisher_pipeline.disable_host: true processors: {{processors}} {{/if}} +auth.aws: +{{#if access_key_id}} + access_key_id: {{access_key_id}} +{{/if}} +{{#if secret_access_key}} + secret_access_key: {{secret_access_key}} +{{/if}} +{{#if session_token}} + session_token: {{session_token}} +{{/if}} +{{#if shared_credential_file}} + shared_credential_file: {{shared_credential_file}} +{{/if}} +{{#if credential_profile_name}} + credential_profile_name: {{credential_profile_name}} +{{/if}} +{{#if role_arn}} + role_arn: {{role_arn}} +{{/if}} +{{#if external_id}} + external_id: {{external_id}} +{{/if}} +{{#if assume_role_duration}} + assume_role.duration: {{assume_role_duration}} +{{/if}} +{{#if assume_role_expiry_window}} + assume_role.expiry_window: {{assume_role_expiry_window}} +{{/if}} +{{#if supports_cloud_connectors}} + use_cloud_connectors: {{supports_cloud_connectors}} +{{/if}} diff --git a/packages/aws/data_stream/guardduty/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/guardduty/elasticsearch/ingest_pipeline/default.yml index 4f5a5de3ee0..4549dd07ef2 100644 --- a/packages/aws/data_stream/guardduty/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/guardduty/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,7 @@ description: Pipeline for processing Amazon GuardDuty Findings logs. processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - remove: @@ -16,31 +17,39 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_event_kind field: event.kind value: [event] - set: + tag: set_event_type field: event.type value: [info] - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original field: event.original target_field: json on_failure: - append: + tag: append_error_message field: error.message value: '{{{_ingest.on_failure_message}}}' - drop: + tag: drop if: ctx.json?.findings != null && ctx.json.findings.isEmpty() - fingerprint: + tag: fingerprint fields: - json.updatedAt - json.id @@ -50,18 +59,22 @@ processors: target_field: _id ignore_missing: true - rename: + tag: rename_json_accountid field: json.accountId target_field: aws.guardduty.account_id ignore_missing: true - rename: + tag: rename_json_arn field: json.arn target_field: aws.guardduty.arn ignore_missing: true - rename: + tag: rename_json_confidence field: json.confidence target_field: aws.guardduty.confidence ignore_missing: true - date: + tag: date_json_createdat field: json.createdAt target_field: aws.guardduty.created_at formats: @@ -71,84 +84,104 @@ processors: if: ctx.json?.createdAt != null on_failure: - append: + tag: append_error_message_1 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_description field: json.description target_field: aws.guardduty.description ignore_missing: true - rename: + tag: rename_json_id field: json.id target_field: aws.guardduty.id ignore_missing: true - rename: + tag: rename_json_partition field: json.partition target_field: aws.guardduty.partition ignore_missing: true - rename: + tag: rename_json_region field: json.region target_field: aws.guardduty.region ignore_missing: true - rename: + tag: rename_json_resource_accesskeydetails_accesskeyid field: json.resource.accessKeyDetails.accessKeyId target_field: aws.guardduty.resource.access_key_details.accesskey_id ignore_missing: true - rename: + tag: rename_json_resource_accesskeydetails_usertype field: json.resource.accessKeyDetails.userType target_field: aws.guardduty.resource.access_key_details.user.type ignore_missing: true - rename: + tag: rename_json_resource_accesskeydetails_principalid field: json.resource.accessKeyDetails.principalId target_field: aws.guardduty.resource.access_key_details.principal_id ignore_missing: true - append: + tag: append_related_user field: related.user value: '{{{aws.guardduty.resource.access_key_details.principal_id}}}' if: ctx.aws?.guardduty?.resource?.access_key_details?.principal_id != null allow_duplicates: false - rename: + tag: rename_json_resource_accesskeydetails_username field: json.resource.accessKeyDetails.userName target_field: aws.guardduty.resource.access_key_details.user.name ignore_missing: true - append: + tag: append_related_user_1 field: related.user value: '{{{aws.guardduty.resource.access_key_details.user.name}}}' if: ctx.aws?.guardduty?.resource?.access_key_details?.user?.name != null allow_duplicates: false - rename: + tag: rename_json_resource_containerdetails_containerruntime field: json.resource.containerDetails.containerRuntime target_field: aws.guardduty.resource.container_details.container_runtime ignore_missing: true - rename: + tag: rename_json_resource_containerdetails_id field: json.resource.containerDetails.id target_field: aws.guardduty.resource.container_details.id ignore_missing: true - rename: + tag: rename_json_resource_containerdetails_image field: json.resource.containerDetails.image target_field: aws.guardduty.resource.container_details.image.value ignore_missing: true - rename: + tag: rename_json_resource_containerdetails_imageprefix field: json.resource.containerDetails.imagePrefix target_field: aws.guardduty.resource.container_details.image.prefix ignore_missing: true - rename: + tag: rename_json_resource_containerdetails_name field: json.resource.containerDetails.name target_field: aws.guardduty.resource.container_details.name ignore_missing: true - convert: + tag: convert_json_resource_containerdetails_securitycontext_privileged field: json.resource.containerDetails.securityContext.privileged target_field: aws.guardduty.resource.container_details.security_context.privileged type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_2 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_container_security_context_privileged field: container.security_context.privileged copy_from: aws.guardduty.resource.container_details.security_context.privileged ignore_empty_value: true - foreach: + tag: foreach_json_resource_containerdetails_volumemounts field: json.resource.containerDetails.volumeMounts if: ctx.json?.resource?.containerDetails?.volumeMounts instanceof List processor: @@ -157,10 +190,12 @@ processors: target_field: _ingest._value.mount_path ignore_missing: true - rename: + tag: rename_json_resource_containerdetails_volumemounts field: json.resource.containerDetails.volumeMounts target_field: aws.guardduty.resource.container_details.volume_mounts ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_scannedvolumedetails field: json.resource.ebsVolumeDetails.scannedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.scannedVolumeDetails instanceof List processor: @@ -169,6 +204,7 @@ processors: target_field: _ingest._value.device_name ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_scannedvolumedetails_1 field: json.resource.ebsVolumeDetails.scannedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.scannedVolumeDetails instanceof List processor: @@ -177,6 +213,7 @@ processors: target_field: _ingest._value.encryption_type ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_scannedvolumedetails_2 field: json.resource.ebsVolumeDetails.scannedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.scannedVolumeDetails instanceof List processor: @@ -185,6 +222,7 @@ processors: target_field: _ingest._value.kmskey_arn ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_scannedvolumedetails_3 field: json.resource.ebsVolumeDetails.scannedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.scannedVolumeDetails instanceof List processor: @@ -193,6 +231,7 @@ processors: target_field: _ingest._value.snapshot_arn ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_scannedvolumedetails_4 field: json.resource.ebsVolumeDetails.scannedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.scannedVolumeDetails instanceof List processor: @@ -201,6 +240,7 @@ processors: target_field: _ingest._value.volume.arn ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_scannedvolumedetails_5 field: json.resource.ebsVolumeDetails.scannedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.scannedVolumeDetails instanceof List processor: @@ -211,9 +251,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resource_ebsvolumedetails_scannedvolumedetails_6 field: json.resource.ebsVolumeDetails.scannedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.scannedVolumeDetails instanceof List processor: @@ -222,6 +264,7 @@ processors: target_field: _ingest._value.volume.type ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_scannedvolumedetails_7 field: json.resource.ebsVolumeDetails.scannedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.scannedVolumeDetails instanceof List processor: @@ -229,10 +272,12 @@ processors: field: _ingest._value.volumeSizeInGB ignore_missing: true - rename: + tag: rename_json_resource_ebsvolumedetails_scannedvolumedetails field: json.resource.ebsVolumeDetails.scannedVolumeDetails target_field: aws.guardduty.resource.ebs_volume_details.scanned_volume_details ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_skippedvolumedetails field: json.resource.ebsVolumeDetails.skippedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.skippedVolumeDetails instanceof List processor: @@ -241,6 +286,7 @@ processors: target_field: _ingest._value.device_name ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_skippedvolumedetails_1 field: json.resource.ebsVolumeDetails.skippedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.skippedVolumeDetails instanceof List processor: @@ -249,6 +295,7 @@ processors: target_field: _ingest._value.encryption_type ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_skippedvolumedetails_2 field: json.resource.ebsVolumeDetails.skippedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.skippedVolumeDetails instanceof List processor: @@ -257,6 +304,7 @@ processors: target_field: _ingest._value.kmskey_arn ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_skippedvolumedetails_3 field: json.resource.ebsVolumeDetails.skippedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.skippedVolumeDetails instanceof List processor: @@ -265,6 +313,7 @@ processors: target_field: _ingest._value.snapshot_arn ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_skippedvolumedetails_4 field: json.resource.ebsVolumeDetails.skippedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.skippedVolumeDetails instanceof List processor: @@ -273,6 +322,7 @@ processors: target_field: _ingest._value.volume.arn ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_skippedvolumedetails_5 field: json.resource.ebsVolumeDetails.skippedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.skippedVolumeDetails instanceof List processor: @@ -283,9 +333,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resource_ebsvolumedetails_skippedvolumedetails_6 field: json.resource.ebsVolumeDetails.skippedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.skippedVolumeDetails instanceof List processor: @@ -294,6 +346,7 @@ processors: target_field: _ingest._value.volume.type ignore_missing: true - foreach: + tag: foreach_json_resource_ebsvolumedetails_skippedvolumedetails_7 field: json.resource.ebsVolumeDetails.skippedVolumeDetails if: ctx.json?.resource?.ebsVolumeDetails?.skippedVolumeDetails instanceof List processor: @@ -301,57 +354,70 @@ processors: field: _ingest._value.volumeSizeInGB ignore_missing: true - rename: + tag: rename_json_resource_ebsvolumedetails_skippedvolumedetails field: json.resource.ebsVolumeDetails.skippedVolumeDetails target_field: aws.guardduty.resource.ebs_volume_details.skipped_volume_details ignore_missing: true - convert: + tag: convert_json_resource_ecsclusterdetails_activeservicescount field: json.resource.ecsClusterDetails.activeServicesCount target_field: aws.guardduty.resource.ecs_cluster_details.active_services_count type: long ignore_missing: true on_failure: - append: + tag: append_error_message_5 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_resource_ecsclusterdetails_arn field: json.resource.ecsClusterDetails.arn target_field: aws.guardduty.resource.ecs_cluster_details.arn ignore_missing: true - rename: + tag: rename_json_resource_ecsclusterdetails_name field: json.resource.ecsClusterDetails.name target_field: aws.guardduty.resource.ecs_cluster_details.name ignore_missing: true - convert: + tag: convert_json_resource_ecsclusterdetails_registeredcontainerinstancescount field: json.resource.ecsClusterDetails.registeredContainerInstancesCount target_field: aws.guardduty.resource.ecs_cluster_details.registered_container_instances_count type: long ignore_missing: true on_failure: - append: + tag: append_error_message_6 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_resource_ecsclusterdetails_runningtaskscount field: json.resource.ecsClusterDetails.runningTasksCount target_field: aws.guardduty.resource.ecs_cluster_details.running_tasks_count type: long ignore_missing: true on_failure: - append: + tag: append_error_message_7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_resource_ecsclusterdetails_status field: json.resource.ecsClusterDetails.status target_field: aws.guardduty.resource.ecs_cluster_details.status ignore_missing: true - rename: + tag: rename_json_resource_ecsclusterdetails_tags field: json.resource.ecsClusterDetails.tags target_field: aws.guardduty.resource.ecs_cluster_details.tags ignore_missing: true - rename: + tag: rename_json_resource_ecsclusterdetails_taskdetails_arn field: json.resource.ecsClusterDetails.taskDetails.arn target_field: aws.guardduty.resource.ecs_cluster_details.task_details.arn ignore_missing: true - foreach: + tag: foreach_json_resource_ecsclusterdetails_taskdetails_containers field: json.resource.ecsClusterDetails.taskDetails.containers if: ctx.json?.resource?.ecsClusterDetails?.taskDetails?.containers instanceof List processor: @@ -360,6 +426,7 @@ processors: target_field: _ingest._value.container_runtime ignore_missing: true - foreach: + tag: foreach_json_resource_ecsclusterdetails_taskdetails_containers_1 field: json.resource.ecsClusterDetails.taskDetails.containers if: ctx.json?.resource?.ecsClusterDetails?.taskDetails?.containers instanceof List processor: @@ -368,6 +435,7 @@ processors: target_field: _ingest._value.image.value ignore_missing: true - foreach: + tag: foreach_json_resource_ecsclusterdetails_taskdetails_containers_2 field: json.resource.ecsClusterDetails.taskDetails.containers if: ctx.json?.resource?.ecsClusterDetails?.taskDetails?.containers instanceof List processor: @@ -376,6 +444,7 @@ processors: target_field: _ingest._value.image.prefix ignore_missing: true - foreach: + tag: foreach_json_resource_ecsclusterdetails_taskdetails_containers_3 field: json.resource.ecsClusterDetails.taskDetails.containers if: ctx.json?.resource?.ecsClusterDetails?.taskDetails?.containers instanceof List processor: @@ -386,9 +455,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resource_ecsclusterdetails_taskdetails_containers_4 field: json.resource.ecsClusterDetails.taskDetails.containers if: ctx.json?.resource?.ecsClusterDetails?.taskDetails?.containers instanceof List processor: @@ -401,6 +472,7 @@ processors: target_field: _ingest._value.mount_path ignore_missing: true - foreach: + tag: foreach_json_resource_ecsclusterdetails_taskdetails_containers_5 field: json.resource.ecsClusterDetails.taskDetails.containers if: ctx.json?.resource?.ecsClusterDetails?.taskDetails?.containers instanceof List processor: @@ -409,6 +481,7 @@ processors: target_field: _ingest._value.volume_mounts ignore_missing: true - foreach: + tag: foreach_json_resource_ecsclusterdetails_taskdetails_containers_6 field: json.resource.ecsClusterDetails.taskDetails.containers if: ctx.json?.resource?.ecsClusterDetails?.taskDetails?.containers instanceof List processor: @@ -416,18 +489,22 @@ processors: field: _ingest._value.securityContext.privileged ignore_missing: true - rename: + tag: rename_json_resource_ecsclusterdetails_taskdetails_containers field: json.resource.ecsClusterDetails.taskDetails.containers target_field: aws.guardduty.resource.ecs_cluster_details.task_details.containers ignore_missing: true - rename: + tag: rename_json_resource_ecsclusterdetails_taskdetails_definitionarn field: json.resource.ecsClusterDetails.taskDetails.definitionArn target_field: aws.guardduty.resource.ecs_cluster_details.task_details.definitionarn ignore_missing: true - rename: + tag: rename_json_resource_ecsclusterdetails_taskdetails_group field: json.resource.ecsClusterDetails.taskDetails.group target_field: aws.guardduty.resource.ecs_cluster_details.task_details.group ignore_missing: true - date: + tag: date_json_resource_ecsclusterdetails_taskdetails_startedat field: json.resource.ecsClusterDetails.taskDetails.startedAt target_field: aws.guardduty.resource.ecs_cluster_details.task_details.started_at formats: @@ -437,17 +514,21 @@ processors: if: ctx.json?.resource?.ecsClusterDetails?.taskDetails?.startedAt != null on_failure: - append: + tag: append_error_message_9 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_resource_ecsclusterdetails_taskdetails_startedby field: json.resource.ecsClusterDetails.taskDetails.startedBy target_field: aws.guardduty.resource.ecs_cluster_details.task_details.started_by ignore_missing: true - rename: + tag: rename_json_resource_ecsclusterdetails_taskdetails_tags field: json.resource.ecsClusterDetails.taskDetails.tags target_field: aws.guardduty.resource.ecs_cluster_details.task_details.tags ignore_missing: true - date: + tag: date_json_resource_ecsclusterdetails_taskdetails_createdat field: json.resource.ecsClusterDetails.taskDetails.createdAt target_field: aws.guardduty.resource.ecs_cluster_details.task_details.created_at formats: @@ -457,13 +538,16 @@ processors: if: ctx.json?.resource?.ecsClusterDetails?.taskDetails?.createdAt != null on_failure: - append: + tag: append_error_message_10 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_resource_ecsclusterdetails_taskdetails_version field: json.resource.ecsClusterDetails.taskDetails.version target_field: aws.guardduty.resource.ecs_cluster_details.task_details.version ignore_missing: true - foreach: + tag: foreach_json_resource_ecsclusterdetails_taskdetails_volumes field: json.resource.ecsClusterDetails.taskDetails.volumes if: ctx.json?.resource?.ecsClusterDetails?.taskDetails?.volumes instanceof List processor: @@ -472,14 +556,17 @@ processors: target_field: _ingest._value.host_path ignore_missing: true - rename: + tag: rename_json_resource_ecsclusterdetails_taskdetails_volumes field: json.resource.ecsClusterDetails.taskDetails.volumes target_field: aws.guardduty.resource.ecs_cluster_details.task_details.volumes ignore_missing: true - rename: + tag: rename_json_resource_eksclusterdetails_arn field: json.resource.eksClusterDetails.arn target_field: aws.guardduty.resource.eks_cluster_details.arn ignore_missing: true - date: + tag: date_json_resource_eksclusterdetails_createdat field: json.resource.eksClusterDetails.createdAt target_field: aws.guardduty.resource.eks_cluster_details.created_at formats: @@ -489,53 +576,66 @@ processors: if: ctx.json?.resource?.eksClusterDetails?.createdAt != null on_failure: - append: + tag: append_error_message_11 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_resource_eksclusterdetails_name field: json.resource.eksClusterDetails.name target_field: aws.guardduty.resource.eks_cluster_details.name ignore_missing: true - rename: + tag: rename_json_resource_eksclusterdetails_status field: json.resource.eksClusterDetails.status target_field: aws.guardduty.resource.eks_cluster_details.status ignore_missing: true - rename: + tag: rename_json_resource_eksclusterdetails_tags field: json.resource.eksClusterDetails.tags target_field: aws.guardduty.resource.eks_cluster_details.tags ignore_missing: true - rename: + tag: rename_json_resource_eksclusterdetails_vpcid field: json.resource.eksClusterDetails.vpcId target_field: aws.guardduty.resource.eks_cluster_details.vpcid ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_availabilityzone field: json.resource.instanceDetails.availabilityZone target_field: aws.guardduty.resource.instance_details.availability_zone ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_iaminstanceprofile field: json.resource.instanceDetails.iamInstanceProfile target_field: aws.guardduty.resource.instance_details.iaminstance_profile ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_imagedescription field: json.resource.instanceDetails.imageDescription target_field: aws.guardduty.resource.instance_details.image.description ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_imageid field: json.resource.instanceDetails.imageId target_field: aws.guardduty.resource.instance_details.image.id ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_instanceid field: json.resource.instanceDetails.instanceId target_field: aws.guardduty.resource.instance_details.instance.id ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_instancestate field: json.resource.instanceDetails.instanceState target_field: aws.guardduty.resource.instance_details.instance.state ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_instancetype field: json.resource.instanceDetails.instanceType target_field: aws.guardduty.resource.instance_details.instance.type ignore_missing: true - date: + tag: date_json_resource_instancedetails_launchtime field: json.resource.instanceDetails.launchTime target_field: aws.guardduty.resource.instance_details.launch_time formats: @@ -545,9 +645,11 @@ processors: if: ctx.json?.resource?.instanceDetails?.launchTime != null on_failure: - append: + tag: append_error_message_12 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -561,11 +663,14 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value field: _ingest._value - append: + tag: append_error_message_13 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_1 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -578,6 +683,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_2 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -586,6 +692,7 @@ processors: target_field: _ingest._value.ipv6_addresses ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_3 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -594,6 +701,7 @@ processors: target_field: _ingest._value.network_interface_id ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_4 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -602,6 +710,7 @@ processors: target_field: _ingest._value.private.dns_name ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_5 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -612,9 +721,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_14 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_6 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -623,6 +734,7 @@ processors: value: '{{{_ingest._value.private.ip_address}}}' allow_duplicates: false - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_7 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -637,9 +749,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_15 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_8 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -652,6 +766,7 @@ processors: value: '{{{_ingest._value.private.ip_address}}}' allow_duplicates: false - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_9 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -664,6 +779,7 @@ processors: target_field: _ingest._value.private.dns_name ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_10 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -675,6 +791,7 @@ processors: field: _ingest._value.privateIpAddress ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_11 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -683,6 +800,7 @@ processors: target_field: _ingest._value.private.ip_addresses ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_12 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -691,6 +809,7 @@ processors: target_field: _ingest._value.public.dns_name ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_13 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -701,9 +820,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_16 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_14 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -712,6 +833,7 @@ processors: value: '{{{_ingest._value.public.ip}}}' allow_duplicates: false - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_15 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -724,6 +846,7 @@ processors: target_field: _ingest._value.group.id ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_16 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -736,6 +859,7 @@ processors: target_field: _ingest._value.group.name ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_17 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -744,6 +868,7 @@ processors: target_field: _ingest._value.security_groups ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_18 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -752,6 +877,7 @@ processors: target_field: _ingest._value.subnet_id ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_19 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -760,6 +886,7 @@ processors: target_field: _ingest._value.vpc_id ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_networkinterfaces_20 field: json.resource.instanceDetails.networkInterfaces if: ctx.json?.resource?.instanceDetails?.networkInterfaces instanceof List processor: @@ -769,18 +896,22 @@ processors: - _ingest._value.publicIp ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_networkinterfaces field: json.resource.instanceDetails.networkInterfaces target_field: aws.guardduty.resource.instance_details.network_interfaces ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_outpostarn field: json.resource.instanceDetails.outpostArn target_field: aws.guardduty.resource.instance_details.outpost_arn ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_platform field: json.resource.instanceDetails.platform target_field: aws.guardduty.resource.instance_details.platform ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_productcodes field: json.resource.instanceDetails.productCodes if: ctx.json?.resource?.instanceDetails?.productCodes instanceof List processor: @@ -789,6 +920,7 @@ processors: target_field: _ingest._value.product_code.id ignore_missing: true - foreach: + tag: foreach_json_resource_instancedetails_productcodes_1 field: json.resource.instanceDetails.productCodes if: ctx.json?.resource?.instanceDetails?.productCodes instanceof List processor: @@ -797,36 +929,44 @@ processors: target_field: _ingest._value.product_code.type ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_productcodes field: json.resource.instanceDetails.productCodes target_field: aws.guardduty.resource.instance_details.product_codes ignore_missing: true - rename: + tag: rename_json_resource_instancedetails_tags field: json.resource.instanceDetails.tags target_field: aws.guardduty.resource.instance_details.tags ignore_missing: true - rename: + tag: rename_json_resource_kubernetesdetails_kubernetesuserdetails_uid field: json.resource.kubernetesDetails.kubernetesUserDetails.uid target_field: aws.guardduty.resource.kubernetes_details.kubernetes_user_details.uid ignore_missing: true - append: + tag: append_related_user_2 field: related.user value: '{{{aws.guardduty.resource.kubernetes_details.kubernetes_user_details.uid}}}' if: ctx.aws?.guardduty?.resource?.kubernetes_details?.kubernetes_user_details?.uid != null allow_duplicates: false - rename: + tag: rename_json_resource_kubernetesdetails_kubernetesuserdetails_username field: json.resource.kubernetesDetails.kubernetesUserDetails.username target_field: aws.guardduty.resource.kubernetes_details.kubernetes_user_details.user_name ignore_missing: true - append: + tag: append_related_user_3 field: related.user value: '{{{aws.guardduty.resource.kubernetes_details.kubernetes_user_details.user_name}}}' if: ctx.aws?.guardduty?.resource?.kubernetes_details?.kubernetes_user_details?.user_name != null allow_duplicates: false - rename: + tag: rename_json_resource_kubernetesdetails_kubernetesuserdetails_groups field: json.resource.kubernetesDetails.kubernetesUserDetails.groups target_field: aws.guardduty.resource.kubernetes_details.kubernetes_user_details.groups ignore_missing: true - foreach: + tag: foreach_json_resource_kubernetesdetails_kubernetesworkloaddetails_containers field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.containers if: ctx.json?.resource?.kubernetesDetails?.kubernetesWorkloadDetails?.containers instanceof List processor: @@ -835,6 +975,7 @@ processors: target_field: _ingest._value.container_runtime ignore_missing: true - foreach: + tag: foreach_json_resource_kubernetesdetails_kubernetesworkloaddetails_containers_1 field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.containers if: ctx.json?.resource?.kubernetesDetails?.kubernetesWorkloadDetails?.containers instanceof List processor: @@ -843,6 +984,7 @@ processors: target_field: _ingest._value.image.value ignore_missing: true - foreach: + tag: foreach_json_resource_kubernetesdetails_kubernetesworkloaddetails_containers_2 field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.containers if: ctx.json?.resource?.kubernetesDetails?.kubernetesWorkloadDetails?.containers instanceof List processor: @@ -851,6 +993,7 @@ processors: target_field: _ingest._value.image.prefix ignore_missing: true - foreach: + tag: foreach_json_resource_kubernetesdetails_kubernetesworkloaddetails_containers_3 field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.containers if: ctx.json?.resource?.kubernetesDetails?.kubernetesWorkloadDetails?.containers instanceof List processor: @@ -861,9 +1004,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_17 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resource_kubernetesdetails_kubernetesworkloaddetails_containers_4 field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.containers if: ctx.json?.resource?.kubernetesDetails?.kubernetesWorkloadDetails?.containers instanceof List processor: @@ -876,6 +1021,7 @@ processors: target_field: _ingest._value.mount_path ignore_missing: true - foreach: + tag: foreach_json_resource_kubernetesdetails_kubernetesworkloaddetails_containers_5 field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.containers if: ctx.json?.resource?.kubernetesDetails?.kubernetesWorkloadDetails?.containers instanceof List processor: @@ -884,6 +1030,7 @@ processors: target_field: _ingest._value.volume_mounts ignore_missing: true - foreach: + tag: foreach_json_resource_kubernetesdetails_kubernetesworkloaddetails_containers_6 field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.containers if: ctx.json?.resource?.kubernetesDetails?.kubernetesWorkloadDetails?.containers instanceof List processor: @@ -891,35 +1038,43 @@ processors: field: _ingest._value.securityContext.privileged ignore_missing: true - rename: + tag: rename_json_resource_kubernetesdetails_kubernetesworkloaddetails_containers field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.containers target_field: aws.guardduty.resource.kubernetes_details.kubernetes_workload_details.containers ignore_missing: true - convert: + tag: convert_json_resource_kubernetesdetails_kubernetesworkloaddetails_hostnetwork field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.hostNetwork target_field: aws.guardduty.resource.kubernetes_details.kubernetes_workload_details.host_network type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_18 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_resource_kubernetesdetails_kubernetesworkloaddetails_name field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.name target_field: aws.guardduty.resource.kubernetes_details.kubernetes_workload_details.name ignore_missing: true - rename: + tag: rename_json_resource_kubernetesdetails_kubernetesworkloaddetails_namespace field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.namespace target_field: aws.guardduty.resource.kubernetes_details.kubernetes_workload_details.name_space ignore_missing: true - rename: + tag: rename_json_resource_kubernetesdetails_kubernetesworkloaddetails_type field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.type target_field: aws.guardduty.resource.kubernetes_details.kubernetes_workload_details.type ignore_missing: true - rename: + tag: rename_json_resource_kubernetesdetails_kubernetesworkloaddetails_uid field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.uid target_field: aws.guardduty.resource.kubernetes_details.kubernetes_workload_details.uid ignore_missing: true - foreach: + tag: foreach_json_resource_kubernetesdetails_kubernetesworkloaddetails_volumes field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.volumes if: ctx.json?.resource?.kubernetesDetails?.kubernetesWorkloadDetails?.volumes instanceof List processor: @@ -928,64 +1083,79 @@ processors: target_field: _ingest._value.host_path ignore_missing: true - rename: + tag: rename_json_resource_kubernetesdetails_kubernetesworkloaddetails_volumes field: json.resource.kubernetesDetails.kubernetesWorkloadDetails.volumes target_field: aws.guardduty.resource.kubernetes_details.kubernetes_workload_details.volumes ignore_missing: true - rename: + tag: rename_json_resource_rdsdbinstancedetails_dbinstanceidentifier field: json.resource.rdsDbInstanceDetails.dbInstanceIdentifier target_field: aws.guardduty.resource.rdsdb_instance_details.instance_identifier ignore_missing: true - rename: + tag: rename_json_resource_rdsdbinstancedetails_engine field: json.resource.rdsDbInstanceDetails.engine target_field: aws.guardduty.resource.rdsdb_instance_details.engine ignore_missing: true - convert: + tag: convert_json_resource_rdsdbinstancedetails_engineversion field: json.resource.rdsDbInstanceDetails.engineVersion target_field: aws.guardduty.resource.rdsdb_instance_details.engine_version type: string ignore_missing: true on_failure: - append: + tag: append_error_message_19 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_resource_rdsdbinstancedetails_dbclusteridentifier field: json.resource.rdsDbInstanceDetails.dbClusterIdentifier target_field: aws.guardduty.resource.rdsdb_instance_details.cluster_identifier ignore_missing: true - rename: + tag: rename_json_resource_rdsdbinstancedetails_dbinstancearn field: json.resource.rdsDbInstanceDetails.dbInstanceArn target_field: aws.guardduty.resource.rdsdb_instance_details.instance_arn ignore_missing: true - rename: + tag: rename_json_resource_rdsdbuserdetails_user field: json.resource.rdsDbUserDetails.user target_field: aws.guardduty.resource.rdsdb_user_details.user ignore_missing: true - append: + tag: append_related_user_4 field: related.user value: '{{{aws.guardduty.resource.rdsdb_user_details.user}}}' if: ctx.aws?.guardduty?.resource?.rdsdb_user_details?.user != null allow_duplicates: false - rename: + tag: rename_json_resource_rdsdbuserdetails_application field: json.resource.rdsDbUserDetails.application target_field: aws.guardduty.resource.rdsdb_user_details.application ignore_missing: true - rename: + tag: rename_json_resource_rdsdbuserdetails_database field: json.resource.rdsDbUserDetails.database target_field: aws.guardduty.resource.rdsdb_user_details.database ignore_missing: true - rename: + tag: rename_json_resource_rdsdbuserdetails_ssl field: json.resource.rdsDbUserDetails.ssl target_field: aws.guardduty.resource.rdsdb_user_details.ssl ignore_missing: true - rename: + tag: rename_json_resource_rdsdbuserdetails_authmethod field: json.resource.rdsDbUserDetails.authMethod target_field: aws.guardduty.resource.rdsdb_user_details.auth_method ignore_missing: true - rename: + tag: rename_json_resource_resourcetype field: json.resource.resourceType target_field: aws.guardduty.resource.type ignore_missing: true - foreach: + tag: foreach_json_resource_s3bucketdetails field: json.resource.s3BucketDetails if: ctx.json?.resource?.s3BucketDetails instanceof List processor: @@ -998,6 +1168,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' ignore_failure: true - foreach: + tag: foreach_json_resource_s3bucketdetails_1 field: json.resource.s3BucketDetails if: ctx.json?.resource?.s3BucketDetails instanceof List processor: @@ -1006,6 +1177,7 @@ processors: target_field: _ingest._value.default_server_side_encryption.encryption_type ignore_missing: true - foreach: + tag: foreach_json_resource_s3bucketdetails_2 field: json.resource.s3BucketDetails if: ctx.json?.resource?.s3BucketDetails instanceof List processor: @@ -1014,6 +1186,7 @@ processors: target_field: _ingest._value.default_server_side_encryption.kms_masterkey_arn ignore_missing: true - foreach: + tag: foreach_json_resource_s3bucketdetails_3 field: json.resource.s3BucketDetails if: ctx.json?.resource?.s3BucketDetails instanceof List processor: @@ -1022,6 +1195,7 @@ processors: target_field: _ingest._value.public_access ignore_missing: true - foreach: + tag: foreach_json_resource_s3bucketdetails_4 field: json.resource.s3BucketDetails if: ctx.json?.resource?.s3BucketDetails instanceof List processor: @@ -1030,6 +1204,7 @@ processors: value: '{{{_ingest._value.owner.id}}}' allow_duplicates: false - foreach: + tag: foreach_json_resource_s3bucketdetails_5 field: json.resource.s3BucketDetails if: ctx.json?.resource?.s3BucketDetails instanceof List processor: @@ -1037,191 +1212,236 @@ processors: field: _ingest._value.createdAt ignore_missing: true - rename: + tag: rename_json_resource_s3bucketdetails field: json.resource.s3BucketDetails target_field: aws.guardduty.resource.s3_bucket_details ignore_missing: true - convert: + tag: convert_json_schemaversion field: json.schemaVersion target_field: aws.guardduty.schema_version type: string ignore_missing: true on_failure: - append: + tag: append_error_message_20 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_action_actiontype field: json.service.action.actionType target_field: aws.guardduty.service.action.type ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_affectedresources field: json.service.action.awsApiCallAction.affectedResources target_field: aws.guardduty.service.action.aws_api_call_action.affected_resources ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_api field: json.service.action.awsApiCallAction.api target_field: aws.guardduty.service.action.aws_api_call_action.api ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_callertype field: json.service.action.awsApiCallAction.callerType target_field: aws.guardduty.service.action.aws_api_call_action.caller_type ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_domaindetails_domain field: json.service.action.awsApiCallAction.domainDetails.domain target_field: aws.guardduty.service.action.aws_api_call_action.domain_details.domain ignore_missing: true - append: + tag: append_related_hosts field: related.hosts value: '{{{aws.guardduty.service.action.aws_api_call_action.domain_details.domain}}}' if: ctx.aws?.guardduty?.service?.action?.aws_api_call_action?.domain_details?.domain != null allow_duplicates: false - rename: + tag: rename_json_service_action_awsapicallaction_errorcode field: json.service.action.awsApiCallAction.errorCode target_field: aws.guardduty.service.action.aws_api_call_action.error_code ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_remoteaccountdetails_accountid field: json.service.action.awsApiCallAction.remoteAccountDetails.accountId target_field: aws.guardduty.service.action.aws_api_call_action.remote_account_details.account_id ignore_missing: true - convert: + tag: convert_json_service_action_awsapicallaction_remoteaccountdetails_affiliated field: json.service.action.awsApiCallAction.remoteAccountDetails.affiliated target_field: aws.guardduty.service.action.aws_api_call_action.remote_account_details.affiliated type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_21 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_action_awsapicallaction_remoteipdetails_city_cityname field: json.service.action.awsApiCallAction.remoteIpDetails.city.cityName target_field: aws.guardduty.service.action.aws_api_call_action.remote_ip_details.city.name ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_remoteipdetails_country_countrycode field: json.service.action.awsApiCallAction.remoteIpDetails.country.countryCode target_field: aws.guardduty.service.action.aws_api_call_action.remote_ip_details.country.code ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_remoteipdetails_country_countryname field: json.service.action.awsApiCallAction.remoteIpDetails.country.countryName target_field: aws.guardduty.service.action.aws_api_call_action.remote_ip_details.country.name ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_remoteipdetails_geolocation field: json.service.action.awsApiCallAction.remoteIpDetails.geoLocation target_field: aws.guardduty.service.action.aws_api_call_action.remote_ip_details.geo_location ignore_missing: true - convert: + tag: convert_json_service_action_awsapicallaction_remoteipdetails_ipaddressv4 field: json.service.action.awsApiCallAction.remoteIpDetails.ipAddressV4 target_field: aws.guardduty.service.action.aws_api_call_action.remote_ip_details.ip_address_v4 type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_22 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip field: related.ip value: '{{{aws.guardduty.service.action.aws_api_call_action.remote_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.aws_api_call_action?.remote_ip_details?.ip_address_v4 != null allow_duplicates: false - rename: + tag: rename_json_service_action_awsapicallaction_remoteipdetails_organization_asn field: json.service.action.awsApiCallAction.remoteIpDetails.organization.asn target_field: aws.guardduty.service.action.aws_api_call_action.remote_ip_details.organization.asn ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_remoteipdetails_organization_asnorg field: json.service.action.awsApiCallAction.remoteIpDetails.organization.asnOrg target_field: aws.guardduty.service.action.aws_api_call_action.remote_ip_details.organization.asnorg ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_remoteipdetails_organization_isp field: json.service.action.awsApiCallAction.remoteIpDetails.organization.isp target_field: aws.guardduty.service.action.aws_api_call_action.remote_ip_details.organization.isp ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_remoteipdetails_organization_org field: json.service.action.awsApiCallAction.remoteIpDetails.organization.org target_field: aws.guardduty.service.action.aws_api_call_action.remote_ip_details.organization.org ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_servicename field: json.service.action.awsApiCallAction.serviceName target_field: aws.guardduty.service.action.aws_api_call_action.service_name ignore_missing: true - rename: + tag: rename_json_service_action_awsapicallaction_useragent field: json.service.action.awsApiCallAction.userAgent target_field: aws.guardduty.service.action.aws_api_call_action.user_agent ignore_missing: true - convert: + tag: convert_json_service_action_dnsrequestaction_blocked field: json.service.action.dnsRequestAction.blocked target_field: aws.guardduty.service.action.dns_request_action.blocked type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_23 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_action_dnsrequestaction_domain field: json.service.action.dnsRequestAction.domain target_field: aws.guardduty.service.action.dns_request_action.domain ignore_missing: true - append: + tag: append_related_hosts_1 field: related.hosts value: '{{{aws.guardduty.service.action.dns_request_action.domain}}}' if: ctx.aws?.guardduty?.service?.action?.dns_request_action?.domain != null allow_duplicates: false - rename: + tag: rename_json_service_action_dnsrequestaction_protocol field: json.service.action.dnsRequestAction.protocol target_field: aws.guardduty.service.action.dns_request_action.protocol ignore_missing: true - rename: + tag: rename_json_service_action_kubernetesapicallaction_parameters field: json.service.action.kubernetesApiCallAction.parameters target_field: aws.guardduty.service.action.kubernetes_api_call_action.parameters ignore_missing: true - rename: + tag: rename_json_service_action_kubernetesapicallaction_remoteipdetails_city_cityname field: json.service.action.kubernetesApiCallAction.remoteIpDetails.city.cityName target_field: aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.city.name ignore_missing: true - rename: + tag: rename_json_service_action_kubernetesapicallaction_remoteipdetails_country_countrycode field: json.service.action.kubernetesApiCallAction.remoteIpDetails.country.countryCode target_field: aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.country.code ignore_missing: true - rename: + tag: rename_json_service_action_kubernetesapicallaction_remoteipdetails_country_countryname field: json.service.action.kubernetesApiCallAction.remoteIpDetails.country.countryName target_field: aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.country.name ignore_missing: true - rename: + tag: rename_json_service_action_kubernetesapicallaction_remoteipdetails_geolocation field: json.service.action.kubernetesApiCallAction.remoteIpDetails.geoLocation target_field: aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.geo_location ignore_missing: true - convert: + tag: convert_json_service_action_kubernetesapicallaction_remoteipdetails_ipaddressv4 field: json.service.action.kubernetesApiCallAction.remoteIpDetails.ipAddressV4 target_field: aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.ip_address_v4 type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_24 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_1 field: related.ip value: '{{{aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.kubernetes_api_call_action?.remote_ip_details?.ip_address_v4 != null allow_duplicates: false - rename: + tag: rename_json_service_action_kubernetesapicallaction_remoteipdetails_organization_asn field: json.service.action.kubernetesApiCallAction.remoteIpDetails.organization.asn target_field: aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.organization.asn ignore_missing: true - rename: + tag: rename_json_service_action_kubernetesapicallaction_remoteipdetails_organization_asnorg field: json.service.action.kubernetesApiCallAction.remoteIpDetails.organization.asnOrg target_field: aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.organization.asnorg ignore_missing: true - rename: + tag: rename_json_service_action_kubernetesapicallaction_remoteipdetails_organization_isp field: json.service.action.kubernetesApiCallAction.remoteIpDetails.organization.isp target_field: aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.organization.isp ignore_missing: true - rename: + tag: rename_json_service_action_kubernetesapicallaction_remoteipdetails_organization_org field: json.service.action.kubernetesApiCallAction.remoteIpDetails.organization.org target_field: aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.organization.org ignore_missing: true - rename: + tag: rename_json_service_action_kubernetesapicallaction_requesturi field: json.service.action.kubernetesApiCallAction.requestUri target_field: aws.guardduty.service.action.kubernetes_api_call_action.request_uri ignore_missing: true - foreach: + tag: foreach_json_service_action_kubernetesapicallaction_sourceips field: json.service.action.kubernetesApiCallAction.sourceIPs if: ctx.json?.service?.action?.kubernetesApiCallAction?.sourceIPs instanceof List processor: @@ -1231,11 +1451,14 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_1 field: _ingest._value - append: + tag: append_error_message_25 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_service_action_kubernetesapicallaction_sourceips_1 field: json.service.action.kubernetesApiCallAction.sourceIPs if: ctx.json?.service?.action?.kubernetesApiCallAction?.sourceIPs instanceof List processor: @@ -1244,139 +1467,171 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - rename: + tag: rename_json_service_action_kubernetesapicallaction_sourceips field: json.service.action.kubernetesApiCallAction.sourceIPs target_field: aws.guardduty.service.action.kubernetes_api_call_action.source_ips ignore_missing: true - convert: + tag: convert_json_service_action_kubernetesapicallaction_statuscode field: json.service.action.kubernetesApiCallAction.statusCode target_field: aws.guardduty.service.action.kubernetes_api_call_action.status_code type: long ignore_missing: true on_failure: - append: + tag: append_error_message_26 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_action_kubernetesapicallaction_useragent field: json.service.action.kubernetesApiCallAction.userAgent target_field: aws.guardduty.service.action.kubernetes_api_call_action.user_agent ignore_missing: true - rename: + tag: rename_json_service_action_kubernetesapicallaction_verb field: json.service.action.kubernetesApiCallAction.verb target_field: aws.guardduty.service.action.kubernetes_api_call_action.verb ignore_missing: true - convert: + tag: convert_json_service_action_networkconnectionaction_blocked field: json.service.action.networkConnectionAction.blocked target_field: aws.guardduty.service.action.network_connection_action.blocked type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_27 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_action_networkconnectionaction_connectiondirection field: json.service.action.networkConnectionAction.connectionDirection target_field: aws.guardduty.service.action.network_connection_action.connection_direction ignore_missing: true - convert: + tag: convert_json_service_action_networkconnectionaction_localipdetails_ipaddressv4 field: json.service.action.networkConnectionAction.localIpDetails.ipAddressV4 target_field: aws.guardduty.service.action.network_connection_action.local_ip_details.ip_address_v4 type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_28 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_2 field: related.ip value: '{{{aws.guardduty.service.action.network_connection_action.local_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.local_ip_details?.ip_address_v4 != null allow_duplicates: false - convert: + tag: convert_json_service_action_networkconnectionaction_localportdetails_port field: json.service.action.networkConnectionAction.localPortDetails.port target_field: aws.guardduty.service.action.network_connection_action.local_port_details.port.value type: long ignore_missing: true on_failure: - append: + tag: append_error_message_29 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_action_networkconnectionaction_localportdetails_portname field: json.service.action.networkConnectionAction.localPortDetails.portName target_field: aws.guardduty.service.action.network_connection_action.local_port_details.port.name ignore_missing: true - rename: + tag: rename_json_service_action_networkconnectionaction_protocol field: json.service.action.networkConnectionAction.protocol target_field: aws.guardduty.service.action.network_connection_action.transport ignore_missing: true - rename: + tag: rename_json_service_action_networkconnectionaction_remoteipdetails_city_cityname field: json.service.action.networkConnectionAction.remoteIpDetails.city.cityName target_field: aws.guardduty.service.action.network_connection_action.remote_ip_details.city.name ignore_missing: true - rename: + tag: rename_json_service_action_networkconnectionaction_remoteipdetails_country_countrycode field: json.service.action.networkConnectionAction.remoteIpDetails.country.countryCode target_field: aws.guardduty.service.action.network_connection_action.remote_ip_details.country.code ignore_missing: true - rename: + tag: rename_json_service_action_networkconnectionaction_remoteipdetails_country_countryname field: json.service.action.networkConnectionAction.remoteIpDetails.country.countryName target_field: aws.guardduty.service.action.network_connection_action.remote_ip_details.country.name ignore_missing: true - rename: + tag: rename_json_service_action_networkconnectionaction_remoteipdetails_geolocation field: json.service.action.networkConnectionAction.remoteIpDetails.geoLocation target_field: aws.guardduty.service.action.network_connection_action.remote_ip_details.geo_location ignore_missing: true - convert: + tag: convert_json_service_action_networkconnectionaction_remoteipdetails_ipaddressv4 field: json.service.action.networkConnectionAction.remoteIpDetails.ipAddressV4 target_field: aws.guardduty.service.action.network_connection_action.remote_ip_details.ip_address_v4 type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_30 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_3 field: related.ip value: '{{{aws.guardduty.service.action.network_connection_action.remote_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.remote_ip_details?.ip_address_v4 != null allow_duplicates: false - rename: + tag: rename_json_service_action_networkconnectionaction_remoteipdetails_organization_asn field: json.service.action.networkConnectionAction.remoteIpDetails.organization.asn target_field: aws.guardduty.service.action.network_connection_action.remote_ip_details.organization.asn ignore_missing: true - rename: + tag: rename_json_service_action_networkconnectionaction_remoteipdetails_organization_asnorg field: json.service.action.networkConnectionAction.remoteIpDetails.organization.asnOrg target_field: aws.guardduty.service.action.network_connection_action.remote_ip_details.organization.asnorg ignore_missing: true - rename: + tag: rename_json_service_action_networkconnectionaction_remoteipdetails_organization_isp field: json.service.action.networkConnectionAction.remoteIpDetails.organization.isp target_field: aws.guardduty.service.action.network_connection_action.remote_ip_details.organization.isp ignore_missing: true - rename: + tag: rename_json_service_action_networkconnectionaction_remoteipdetails_organization_org field: json.service.action.networkConnectionAction.remoteIpDetails.organization.org target_field: aws.guardduty.service.action.network_connection_action.remote_ip_details.organization.org ignore_missing: true - convert: + tag: convert_json_service_action_networkconnectionaction_remoteportdetails_port field: json.service.action.networkConnectionAction.remotePortDetails.port target_field: aws.guardduty.service.action.network_connection_action.remote_port_details.port.value type: long ignore_missing: true on_failure: - append: + tag: append_error_message_31 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_action_networkconnectionaction_remoteportdetails_portname field: json.service.action.networkConnectionAction.remotePortDetails.portName target_field: aws.guardduty.service.action.network_connection_action.remote_port_details.port.name ignore_missing: true - convert: + tag: convert_json_service_action_portprobeaction_blocked field: json.service.action.portProbeAction.blocked target_field: aws.guardduty.service.action.port_probe_action.blocked type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_32 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1387,9 +1642,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_33 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_1 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1398,6 +1655,7 @@ processors: value: '{{{_ingest._value.local_ip_details.ip_address_v4}}}' allow_duplicates: false - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_2 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1408,9 +1666,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_34 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_3 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1419,6 +1679,7 @@ processors: target_field: _ingest._value.local_port_details.port.name ignore_missing: true - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_4 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1427,6 +1688,7 @@ processors: target_field: _ingest._value.remote_ip_details.city.name ignore_missing: true - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_5 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1435,6 +1697,7 @@ processors: target_field: _ingest._value.remote_ip_details.country.code ignore_missing: true - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_6 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1443,6 +1706,7 @@ processors: target_field: _ingest._value.remote_ip_details.country.name ignore_missing: true - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_7 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1451,6 +1715,7 @@ processors: target_field: _ingest._value.remote_ip_details.geo_location ignore_missing: true - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_8 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1461,9 +1726,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_35 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_9 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1472,6 +1739,7 @@ processors: value: '{{{_ingest._value.remote_ip_details.ip_address_v4}}}' allow_duplicates: false - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_10 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1480,6 +1748,7 @@ processors: target_field: _ingest._value.remote_ip_details.organization.isp ignore_missing: true - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_11 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1488,6 +1757,7 @@ processors: target_field: _ingest._value.remote_ip_details.organization.org ignore_missing: true - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_12 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1496,6 +1766,7 @@ processors: target_field: _ingest._value.remote_ip_details.organization.asn ignore_missing: true - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_13 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1504,6 +1775,7 @@ processors: target_field: _ingest._value.remote_ip_details.organization.asnorg ignore_missing: true - foreach: + tag: foreach_json_service_action_portprobeaction_portprobedetails_14 field: json.service.action.portProbeAction.portProbeDetails if: ctx.json?.service?.action?.portProbeAction?.portProbeDetails instanceof List processor: @@ -1514,82 +1786,101 @@ processors: - _ingest._value.remoteIpDetails.ipAddressV4 ignore_missing: true - rename: + tag: rename_json_service_action_portprobeaction_portprobedetails field: json.service.action.portProbeAction.portProbeDetails target_field: aws.guardduty.service.action.port_probe_action.port_probe_details ignore_missing: true - rename: + tag: rename_json_service_action_rdsloginattemptaction_remoteipdetails_city_cityname field: json.service.action.rdsLoginAttemptAction.remoteIpDetails.city.cityName target_field: aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.city.name ignore_missing: true - rename: + tag: rename_json_service_action_rdsloginattemptaction_remoteipdetails_country_countrycode field: json.service.action.rdsLoginAttemptAction.remoteIpDetails.country.countryCode target_field: aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.country.code ignore_missing: true - rename: + tag: rename_json_service_action_rdsloginattemptaction_remoteipdetails_country_countryname field: json.service.action.rdsLoginAttemptAction.remoteIpDetails.country.countryName target_field: aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.country.name ignore_missing: true - rename: + tag: rename_json_service_action_rdsloginattemptaction_remoteipdetails_geolocation field: json.service.action.rdsLoginAttemptAction.remoteIpDetails.geoLocation target_field: aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.geo_location ignore_missing: true - convert: + tag: convert_json_service_action_rdsloginattemptaction_remoteipdetails_ipaddressv4 field: json.service.action.rdsLoginAttemptAction.remoteIpDetails.ipAddressV4 target_field: aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.ip_address_v4 type: ip ignore_missing: true on_failure: - append: + tag: append_error_message_36 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_related_ip_4 field: related.ip value: '{{{aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.rds_login_attempt_action?.remote_ip_details?.ip_address_v4 != null allow_duplicates: false - rename: + tag: rename_json_service_action_rdsloginattemptaction_remoteipdetails_organization_asn field: json.service.action.rdsLoginAttemptAction.remoteIpDetails.organization.asn target_field: aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.organization.asn ignore_missing: true - rename: + tag: rename_json_service_action_rdsloginattemptaction_remoteipdetails_organization_asnorg field: json.service.action.rdsLoginAttemptAction.remoteIpDetails.organization.asnOrg target_field: aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.organization.asnorg ignore_missing: true - rename: + tag: rename_json_service_action_rdsloginattemptaction_remoteipdetails_organization_isp field: json.service.action.rdsLoginAttemptAction.remoteIpDetails.organization.isp target_field: aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.organization.isp ignore_missing: true - rename: + tag: rename_json_service_action_rdsloginattemptaction_remoteipdetails_organization_org field: json.service.action.rdsLoginAttemptAction.remoteIpDetails.organization.org target_field: aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.organization.org ignore_missing: true - rename: + tag: rename_json_service_additionalinfo field: json.service.additionalInfo target_field: aws.guardduty.service.additional_info ignore_missing: true - convert: + tag: convert_json_service_archived field: json.service.archived target_field: aws.guardduty.service.archived type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_37 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_service_count field: json.service.count target_field: aws.guardduty.service.count type: long ignore_missing: true on_failure: - append: + tag: append_error_message_38 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_detectorid field: json.service.detectorId target_field: aws.guardduty.service.detector_id ignore_missing: true - date: + tag: date_json_service_ebsvolumescandetails_scancompletedat field: json.service.ebsVolumeScanDetails.scanCompletedAt target_field: aws.guardduty.service.ebs_volume_scan_details.scan.completed_at formats: @@ -1599,71 +1890,87 @@ processors: if: ctx.json?.service?.ebsVolumeScanDetails?.scanCompletedAt != null on_failure: - append: + tag: append_error_message_39 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_service_ebsvolumescandetails_scandetections_highestseveritythreatdetails_count field: json.service.ebsVolumeScanDetails.scanDetections.highestSeverityThreatDetails.count target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.highest_severity_threat_details.count type: long ignore_missing: true on_failure: - append: + tag: append_error_message_40 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_ebsvolumescandetails_scandetections_highestseveritythreatdetails_severity field: json.service.ebsVolumeScanDetails.scanDetections.highestSeverityThreatDetails.severity target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.highest_severity_threat_details.severity ignore_missing: true - rename: + tag: rename_json_service_ebsvolumescandetails_scandetections_highestseveritythreatdetails_threatname field: json.service.ebsVolumeScanDetails.scanDetections.highestSeverityThreatDetails.threatName target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.highest_severity_threat_details.threat_name ignore_missing: true - convert: + tag: convert_json_service_ebsvolumescandetails_scandetections_scanneditemcount_files field: json.service.ebsVolumeScanDetails.scanDetections.scannedItemCount.files target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.scanned_item_count.files type: long ignore_missing: true on_failure: - append: + tag: append_error_message_41 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_service_ebsvolumescandetails_scandetections_scanneditemcount_totalgb field: json.service.ebsVolumeScanDetails.scanDetections.scannedItemCount.totalGb target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.scanned_item_count.total_gb type: long ignore_missing: true on_failure: - append: + tag: append_error_message_42 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_service_ebsvolumescandetails_scandetections_scanneditemcount_volumes field: json.service.ebsVolumeScanDetails.scanDetections.scannedItemCount.volumes target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.scanned_item_count.volumes type: long ignore_missing: true on_failure: - append: + tag: append_error_message_43 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_itemcount field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.itemCount target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.threat_detected_by_name.item_count type: long ignore_missing: true on_failure: - append: + tag: append_error_message_44 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_shortened field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.shortened target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.threat_detected_by_name.shortened type: boolean ignore_missing: true on_failure: - append: + tag: append_error_message_45 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_threatnames field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.threatNames if: ctx.json?.service?.ebsVolumeScanDetails?.scanDetections?.threatDetectedByName?.threatNames instanceof List processor: @@ -1676,6 +1983,7 @@ processors: target_field: _ingest._value.file.name ignore_missing: true - foreach: + tag: foreach_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_threatnames_1 field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.threatNames if: ctx.json?.service?.ebsVolumeScanDetails?.scanDetections?.threatDetectedByName?.threatNames instanceof List processor: @@ -1688,6 +1996,7 @@ processors: target_field: _ingest._value.file.path ignore_missing: true - foreach: + tag: foreach_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_threatnames_2 field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.threatNames if: ctx.json?.service?.ebsVolumeScanDetails?.scanDetections?.threatDetectedByName?.threatNames instanceof List processor: @@ -1700,6 +2009,7 @@ processors: value: '{{{_ingest._value.hash}}}' allow_duplicates: false - foreach: + tag: foreach_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_threatnames_3 field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.threatNames if: ctx.json?.service?.ebsVolumeScanDetails?.scanDetections?.threatDetectedByName?.threatNames instanceof List processor: @@ -1712,6 +2022,7 @@ processors: target_field: _ingest._value.volume_arn ignore_missing: true - foreach: + tag: foreach_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_threatnames_4 field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.threatNames if: ctx.json?.service?.ebsVolumeScanDetails?.scanDetections?.threatDetectedByName?.threatNames instanceof List processor: @@ -1720,6 +2031,7 @@ processors: target_field: _ingest._value.file_paths ignore_missing: true - foreach: + tag: foreach_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_threatnames_5 field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.threatNames if: ctx.json?.service?.ebsVolumeScanDetails?.scanDetections?.threatDetectedByName?.threatNames instanceof List processor: @@ -1730,9 +2042,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_46 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_threatnames_6 field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.threatNames if: ctx.json?.service?.ebsVolumeScanDetails?.scanDetections?.threatDetectedByName?.threatNames instanceof List processor: @@ -1741,32 +2055,39 @@ processors: - _ingest._value.itemCount ignore_missing: true - rename: + tag: rename_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_threatnames field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.threatNames target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.threat_detected_by_name.threat_names ignore_missing: true - convert: + tag: convert_json_service_ebsvolumescandetails_scandetections_threatdetectedbyname_uniquethreatnamecount field: json.service.ebsVolumeScanDetails.scanDetections.threatDetectedByName.uniqueThreatNameCount target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.threat_detected_by_name.unique_threat_name_count type: long ignore_missing: true on_failure: - append: + tag: append_error_message_47 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_service_ebsvolumescandetails_scandetections_threatsdetecteditemcount_files field: json.service.ebsVolumeScanDetails.scanDetections.threatsDetectedItemCount.files target_field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.threats_detected_item_count.files type: long ignore_missing: true on_failure: - append: + tag: append_error_message_48 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_ebsvolumescandetails_scanid field: json.service.ebsVolumeScanDetails.scanId target_field: aws.guardduty.service.ebs_volume_scan_details.scan.id ignore_missing: true - date: + tag: date_json_service_ebsvolumescandetails_scanstartedat field: json.service.ebsVolumeScanDetails.scanStartedAt target_field: aws.guardduty.service.ebs_volume_scan_details.scan.started_at formats: @@ -1776,17 +2097,21 @@ processors: if: ctx.json?.service?.ebsVolumeScanDetails?.scanStartedAt != null on_failure: - append: + tag: append_error_message_49 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_ebsvolumescandetails_sources field: json.service.ebsVolumeScanDetails.sources target_field: aws.guardduty.service.ebs_volume_scan_details.sources ignore_missing: true - rename: + tag: rename_json_service_ebsvolumescandetails_triggerfindingid field: json.service.ebsVolumeScanDetails.triggerFindingId target_field: aws.guardduty.service.ebs_volume_scan_details.trigger_finding_id ignore_missing: true - date: + tag: date_json_service_eventfirstseen field: json.service.eventFirstSeen target_field: aws.guardduty.service.event.first_seen formats: @@ -1796,9 +2121,11 @@ processors: if: ctx.json?.service?.eventFirstSeen != null on_failure: - append: + tag: append_error_message_50 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_service_eventlastseen field: json.service.eventLastSeen target_field: aws.guardduty.service.event.last_seen formats: @@ -1808,9 +2135,11 @@ processors: if: ctx.json?.service?.eventLastSeen != null on_failure: - append: + tag: append_error_message_51 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_service_evidence_threatintelligencedetails field: json.service.evidence.threatIntelligenceDetails if: ctx.json?.service?.evidence?.threatIntelligenceDetails instanceof List processor: @@ -1819,6 +2148,7 @@ processors: target_field: _ingest._value.threat.list_name ignore_missing: true - foreach: + tag: foreach_json_service_evidence_threatintelligencedetails_1 field: json.service.evidence.threatIntelligenceDetails if: ctx.json?.service?.evidence?.threatIntelligenceDetails instanceof List processor: @@ -1827,55 +2157,68 @@ processors: target_field: _ingest._value.threat.names ignore_missing: true - rename: + tag: rename_json_service_evidence_threatintelligencedetails field: json.service.evidence.threatIntelligenceDetails target_field: aws.guardduty.service.evidence.threat_intelligence_details ignore_missing: true - rename: + tag: rename_json_service_featurename field: json.service.featureName target_field: aws.guardduty.service.feature_name ignore_missing: true - rename: + tag: rename_json_service_resourcerole field: json.service.resourceRole target_field: aws.guardduty.service.resource_role ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_addressfamily field: json.service.runtimeDetails.context.addressFamily target_field: aws.guardduty.service.runtime_details.context.address_family ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_commandlineexample field: json.service.runtimeDetails.context.commandLineExample target_field: aws.guardduty.service.runtime_details.context.command_line_example ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_filesystemtype field: json.service.runtimeDetails.context.fileSystemType target_field: aws.guardduty.service.runtime_details.context.file_system_type ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_flags field: json.service.runtimeDetails.context.flags target_field: aws.guardduty.service.runtime_details.context.flags ignore_missing: true - convert: + tag: convert_json_service_runtimedetails_context_ianaprotocolnumber field: json.service.runtimeDetails.context.ianaProtocolNumber target_field: aws.guardduty.service.runtime_details.context.iana_protocol_number type: string ignore_missing: true on_failure: - append: + tag: append_error_message_52 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_runtimedetails_context_ldpreloadvalue field: json.service.runtimeDetails.context.ldPreloadValue target_field: aws.guardduty.service.runtime_details.context.ld_preload ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_librarypath field: json.service.runtimeDetails.context.libraryPath target_field: aws.guardduty.service.runtime_details.context.library_path ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_memoryregions field: json.service.runtimeDetails.context.memoryRegions target_field: aws.guardduty.service.runtime_details.context.memory_regions ignore_missing: true - date: + tag: date_json_service_runtimedetails_context_modifiedat field: json.service.runtimeDetails.context.modifiedAt target_field: aws.guardduty.service.runtime_details.context.modified_at formats: @@ -1885,134 +2228,166 @@ processors: if: ctx.json?.service?.runtimeDetails?.context?.modifiedAt != null on_failure: - append: + tag: append_error_message_53 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_runtimedetails_context_modifyingprocess field: json.service.runtimeDetails.context.modifyingProcess target_field: aws.guardduty.service.runtime_details.context.modifying_process ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_modulefilepath field: json.service.runtimeDetails.context.moduleFilePath target_field: aws.guardduty.service.runtime_details.context.module_file_path ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_modulename field: json.service.runtimeDetails.context.moduleName target_field: aws.guardduty.service.runtime_details.context.module_name ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_modulesha256 field: json.service.runtimeDetails.context.moduleSha256 target_field: aws.guardduty.service.runtime_details.context.module_sha256 ignore_missing: true - append: + tag: append_related_hash field: related.hash value: '{{{aws.guardduty.service.runtime_details.context.module_sha256}}}' if: ctx.aws?.guardduty?.service?.runtime_details?.context?.module_sha256 != null allow_duplicates: false - rename: + tag: rename_json_service_runtimedetails_context_mountsource field: json.service.runtimeDetails.context.mountSource target_field: aws.guardduty.service.runtime_details.context.mount_source ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_mounttarget field: json.service.runtimeDetails.context.mountTarget target_field: aws.guardduty.service.runtime_details.context.mount_target ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_releaseagentpath field: json.service.runtimeDetails.context.releaseAgentPath target_field: aws.guardduty.service.runtime_details.context.release_agent_path ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_runcbinarypath field: json.service.runtimeDetails.context.runcBinaryPath target_field: aws.guardduty.service.runtime_details.context.runc_binary_path ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_scriptpath field: json.service.runtimeDetails.context.scriptPath target_field: aws.guardduty.service.runtime_details.context.script_path ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_servicename field: json.service.runtimeDetails.context.serviceName target_field: aws.guardduty.service.runtime_details.context.service_name ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_shellhistoryfilepath field: json.service.runtimeDetails.context.shellHistoryFilePath target_field: aws.guardduty.service.runtime_details.context.shell_history_file_path ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_socketpath field: json.service.runtimeDetails.context.socketPath target_field: aws.guardduty.service.runtime_details.context.socket_path ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_targetprocess field: json.service.runtimeDetails.context.targetProcess target_field: aws.guardduty.service.runtime_details.context.target_process ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_threatfilepath field: json.service.runtimeDetails.context.threatFilePath target_field: aws.guardduty.service.runtime_details.context.threat_file_path ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_toolcategory field: json.service.runtimeDetails.context.toolCategory target_field: aws.guardduty.service.runtime_details.context.tool_category ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_context_toolname field: json.service.runtimeDetails.context.toolName target_field: aws.guardduty.service.runtime_details.context.tool_name ignore_missing: true - convert: + tag: convert_json_service_runtimedetails_process_euid field: json.service.runtimeDetails.process.euid target_field: aws.guardduty.service.runtime_details.process.euid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_54 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_runtimedetails_process_executablepath field: json.service.runtimeDetails.process.executablePath target_field: aws.guardduty.service.runtime_details.process.executable_path ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_process_executablesha256 field: json.service.runtimeDetails.process.executableSha256 target_field: aws.guardduty.service.runtime_details.process.executable_sha256 ignore_missing: true - append: + tag: append_related_hash_1 field: related.hash value: '{{{aws.guardduty.service.runtime_details.process.executable_sha256}}}' if: ctx.aws?.guardduty?.service?.runtime_details?.process?.executable_sha256 != null allow_duplicates: false - rename: + tag: rename_json_service_runtimedetails_process_lineage field: json.service.runtimeDetails.process.lineage target_field: aws.guardduty.service.runtime_details.process.lineage ignore_missing: true - rename: + tag: rename_json_service_runtimedetails_process_name field: json.service.runtimeDetails.process.name target_field: aws.guardduty.service.runtime_details.process.name ignore_missing: true - convert: + tag: convert_json_service_runtimedetails_process_namespacepid field: json.service.runtimeDetails.process.namespacePid target_field: aws.guardduty.service.runtime_details.process.namespace_pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_55 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_runtimedetails_process_parentuuid field: json.service.runtimeDetails.process.parentUuid target_field: aws.guardduty.service.runtime_details.process.parent_uuid ignore_missing: true - convert: + tag: convert_json_service_runtimedetails_process_pid field: json.service.runtimeDetails.process.pid target_field: aws.guardduty.service.runtime_details.process.pid type: long ignore_missing: true on_failure: - append: + tag: append_error_message_56 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_runtimedetails_process_pwd field: json.service.runtimeDetails.process.pwd target_field: aws.guardduty.service.runtime_details.process.pwd ignore_missing: true - date: + tag: date_json_service_runtimedetails_process_starttime field: json.service.runtimeDetails.process.startTime target_field: aws.guardduty.service.runtime_details.process.start_time formats: @@ -2022,63 +2397,78 @@ processors: if: ctx.json?.service?.runtimeDetails?.process?.startTime != null on_failure: - append: + tag: append_error_message_57 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_runtimedetails_process_user field: json.service.runtimeDetails.process.user target_field: aws.guardduty.service.runtime_details.process.user ignore_missing: true - convert: + tag: convert_json_service_runtimedetails_process_userid field: json.service.runtimeDetails.process.userId target_field: aws.guardduty.service.runtime_details.process.user_id type: long ignore_missing: true on_failure: - append: + tag: append_error_message_58 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_service_runtimedetails_process_uuid field: json.service.runtimeDetails.process.uuid target_field: aws.guardduty.service.runtime_details.process.uuid ignore_missing: true - rename: + tag: rename_json_service_servicename field: json.service.serviceName target_field: aws.guardduty.service.service_name ignore_missing: true - rename: + tag: rename_json_service_userfeedback field: json.service.userFeedback target_field: aws.guardduty.service.user_feedback ignore_missing: true - convert: + tag: convert_json_severity field: json.severity target_field: aws.guardduty.severity.code type: double ignore_missing: true on_failure: - append: + tag: append_error_message_59 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_aws_guardduty_severity_value field: aws.guardduty.severity.value value: 'High' if: ctx.aws?.guardduty?.severity?.code != null && ctx.aws.guardduty.severity.code <= 8.9 && ctx.aws.guardduty.severity.code >= 7.0 - set: + tag: set_aws_guardduty_severity_value_1 field: aws.guardduty.severity.value value: 'Medium' if: ctx.aws?.guardduty?.severity?.code != null && ctx.aws.guardduty.severity.code <= 6.9 && ctx.aws.guardduty.severity.code >= 4.0 - set: + tag: set_aws_guardduty_severity_value_2 field: aws.guardduty.severity.value value: 'Low' if: ctx.aws?.guardduty?.severity?.code != null && ctx.aws.guardduty.severity.code <= 3.9 && ctx.aws.guardduty.severity.code >= 1.0 - rename: + tag: rename_json_title field: json.title target_field: aws.guardduty.title ignore_missing: true - rename: + tag: rename_json_type field: json.type target_field: aws.guardduty.type ignore_missing: true - date: + tag: date_json_updatedat field: json.updatedAt target_field: aws.guardduty.updated_at formats: @@ -2088,33 +2478,41 @@ processors: if: ctx.json?.updatedAt != null on_failure: - append: + tag: append_error_message_60 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_timestamp field: '@timestamp' copy_from: aws.guardduty.updated_at ignore_empty_value: true - set: + tag: set_message field: message copy_from: aws.guardduty.description ignore_empty_value: true - set: + tag: set_cloud_account_id field: cloud.account.id copy_from: aws.guardduty.account_id ignore_empty_value: true - set: + tag: set_cloud_provider field: cloud.provider copy_from: aws.guardduty.partition ignore_empty_value: true - set: + tag: set_cloud_region field: cloud.region copy_from: aws.guardduty.region ignore_empty_value: true - set: + tag: set_cloud_service_name field: cloud.service.name copy_from: aws.guardduty.service.service_name ignore_empty_value: true - foreach: + tag: foreach_aws_guardduty_resource_ecs_cluster_details_task_details_containers field: aws.guardduty.resource.ecs_cluster_details.task_details.containers if: ctx.aws?.guardduty?.resource?.ecs_cluster_details?.task_details?.containers instanceof List processor: @@ -2123,6 +2521,7 @@ processors: value: '{{{_ingest._value.id}}}' allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_resource_ecs_cluster_details_task_details_containers_1 field: aws.guardduty.resource.ecs_cluster_details.task_details.containers if: ctx.aws?.guardduty?.resource?.ecs_cluster_details?.task_details?.containers instanceof List processor: @@ -2131,6 +2530,7 @@ processors: value: '{{{_ingest._value.name}}}' allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_resource_ecs_cluster_details_task_details_containers_2 field: aws.guardduty.resource.ecs_cluster_details.task_details.containers if: ctx.aws?.guardduty?.resource?.ecs_cluster_details?.task_details?.containers instanceof List processor: @@ -2139,36 +2539,43 @@ processors: value: '{{{_ingest._value.container_runtime}}}' allow_duplicates: false - append: + tag: append_source_address field: source.address value: '{{{aws.guardduty.service.action.aws_api_call_action.remote_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.aws_api_call_action?.remote_ip_details?.ip_address_v4 != null allow_duplicates: false - append: + tag: append_source_address_1 field: source.address value: '{{{aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.kubernetes_api_call_action?.remote_ip_details?.ip_address_v4 != null allow_duplicates: false - append: + tag: append_source_address_2 field: source.address value: '{{{aws.guardduty.service.action.network_connection_action.remote_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.connection_direction != null && ctx.aws.guardduty.service.action.network_connection_action.connection_direction.toLowerCase() == 'inbound' allow_duplicates: false - append: + tag: append_destination_address field: destination.address value: '{{{aws.guardduty.service.action.network_connection_action.remote_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.connection_direction != null && ctx.aws.guardduty.service.action.network_connection_action.connection_direction.toLowerCase() == 'outbound' allow_duplicates: false - append: + tag: append_source_address_3 field: source.address value: '{{{aws.guardduty.service.action.network_connection_action.local_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.connection_direction != null && ctx.aws.guardduty.service.action.network_connection_action.connection_direction.toLowerCase() == 'outbound' allow_duplicates: false - append: + tag: append_destination_address_1 field: destination.address value: '{{{aws.guardduty.service.action.network_connection_action.local_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.connection_direction != null && ctx.aws.guardduty.service.action.network_connection_action.connection_direction.toLowerCase() == 'inbound' allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_service_action_port_probe_action_port_probe_details field: aws.guardduty.service.action.port_probe_action.port_probe_details if: ctx.aws?.guardduty?.service?.action?.port_probe_action?.port_probe_details instanceof List processor: @@ -2177,26 +2584,31 @@ processors: value: '{{{_ingest._value.remote_ip_details.ip_address_v4}}}' allow_duplicates: false - append: + tag: append_source_address_4 field: source.address value: '{{{aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.ip_address_v4}}}' if: ctx.aws?.guardduty?.service?.action?.rds_login_attempt_action?.remote_ip_details?.ip_address_v4 != null allow_duplicates: false - append: + tag: append_source_as_number field: source.as.number value: '{{{aws.guardduty.service.action.aws_api_call_action.remote_ip_details.organization.asn}}}' if: ctx.aws?.guardduty?.service?.action?.aws_api_call_action?.remote_ip_details?.organization?.asn != null allow_duplicates: false - append: + tag: append_source_as_number_1 field: source.as.number value: '{{{aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.organization.asn}}}' if: ctx.aws?.guardduty?.service?.action?.kubernetes_api_call_action?.remote_ip_details?.organization?.asn != null allow_duplicates: false - append: + tag: append_source_as_number_2 field: source.as.number value: '{{{aws.guardduty.service.action.network_connection_action.remote_ip_details.organization.asn}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.remote_ip_details?.organization?.asn != null allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_service_action_port_probe_action_port_probe_details_1 field: aws.guardduty.service.action.port_probe_action.port_probe_details if: ctx.aws?.guardduty?.service?.action?.port_probe_action?.port_probe_details instanceof List processor: @@ -2205,11 +2617,13 @@ processors: value: '{{{_ingest._value.remote_ip_details.organization.asn}}}' allow_duplicates: false - append: + tag: append_source_as_number_3 field: source.as.number value: '{{{aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.organization.asn}}}' if: ctx.aws?.guardduty?.service?.action?.rds_login_attempt_action?.remote_ip_details?.organization?.asn != null allow_duplicates: false - foreach: + tag: foreach_source_as_number field: source.as.number if: ctx.source?.as?.number instanceof List processor: @@ -2219,26 +2633,32 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_2 field: _ingest._value - append: + tag: append_error_message_61 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_source_as_organization_name field: source.as.organization.name value: '{{{aws.guardduty.service.action.aws_api_call_action.remote_ip_details.organization.asnorg}}}' if: ctx.aws?.guardduty?.service?.action?.aws_api_call_action?.remote_ip_details?.organization?.asnorg != null allow_duplicates: false - append: + tag: append_source_as_organization_name_1 field: source.as.organization.name value: '{{{aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.organization.asnorg}}}' if: ctx.aws?.guardduty?.service?.action?.kubernetes_api_call_action?.remote_ip_details?.organization?.asnorg != null allow_duplicates: false - append: + tag: append_source_as_organization_name_2 field: source.as.organization.name value: '{{{aws.guardduty.service.action.network_connection_action.remote_ip_details.organization.asnorg}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.remote_ip_details?.organization?.asnorg != null allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_service_action_port_probe_action_port_probe_details_2 field: aws.guardduty.service.action.port_probe_action.port_probe_details if: ctx.aws?.guardduty?.service?.action?.port_probe_action?.port_probe_details instanceof List processor: @@ -2247,26 +2667,31 @@ processors: value: '{{{_ingest._value.remote_ip_details.organization.asnorg}}}' allow_duplicates: false - append: + tag: append_source_as_organization_name_3 field: source.as.organization.name value: '{{{aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.organization.asnorg}}}' if: ctx.aws?.guardduty?.service?.action?.rds_login_attempt_action?.remote_ip_details?.organization?.asnorg != null allow_duplicates: false - append: + tag: append_source_geo_city_name field: source.geo.city_name value: '{{{aws.guardduty.service.action.aws_api_call_action.remote_ip_details.city.name}}}' if: ctx.aws?.guardduty?.service?.action?.aws_api_call_action?.remote_ip_details?.city?.name != null allow_duplicates: false - append: + tag: append_source_geo_city_name_1 field: source.geo.city_name value: '{{{aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.city.name}}}' if: ctx.aws?.guardduty?.service?.action?.kubernetes_api_call_action?.remote_ip_details?.city?.name != null allow_duplicates: false - append: + tag: append_source_geo_city_name_2 field: source.geo.city_name value: '{{{aws.guardduty.service.action.network_connection_action.remote_ip_details.city.name}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.remote_ip_details?.city?.name != null allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_service_action_port_probe_action_port_probe_details_3 field: aws.guardduty.service.action.port_probe_action.port_probe_details if: ctx.aws?.guardduty?.service?.action?.port_probe_action?.port_probe_details instanceof List processor: @@ -2275,26 +2700,31 @@ processors: value: '{{{_ingest._value.remote_ip_details.city.name}}}' allow_duplicates: false - append: + tag: append_source_geo_city_name_3 field: source.geo.city_name value: '{{{aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.city.name}}}' if: ctx.aws?.guardduty?.service?.action?.rds_login_attempt_action?.remote_ip_details?.city?.name != null allow_duplicates: false - append: + tag: append_source_geo_country_iso_code field: source.geo.country_iso_code value: '{{{aws.guardduty.service.action.aws_api_call_action.remote_ip_details.country.code}}}' if: ctx.aws?.guardduty?.service?.action?.aws_api_call_action?.remote_ip_details?.country?.code != null allow_duplicates: false - append: + tag: append_source_geo_country_iso_code_1 field: source.geo.country_iso_code value: '{{{aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.country.code}}}' if: ctx.aws?.guardduty?.service?.action?.kubernetes_api_call_action?.remote_ip_details?.country?.code != null allow_duplicates: false - append: + tag: append_source_geo_country_iso_code_2 field: source.geo.country_iso_code value: '{{{aws.guardduty.service.action.network_connection_action.remote_ip_details.country.code}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.remote_ip_details?.country?.code != null allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_service_action_port_probe_action_port_probe_details_4 field: aws.guardduty.service.action.port_probe_action.port_probe_details if: ctx.aws?.guardduty?.service?.action?.port_probe_action?.port_probe_details instanceof List processor: @@ -2303,26 +2733,31 @@ processors: value: '{{{_ingest._value.remote_ip_details.country.code}}}' allow_duplicates: false - append: + tag: append_source_geo_country_iso_code_3 field: source.geo.country_iso_code value: '{{{aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.country.code}}}' if: ctx.aws?.guardduty?.service?.action?.rds_login_attempt_action?.remote_ip_details?.country?.code != null allow_duplicates: false - append: + tag: append_source_geo_country_name field: source.geo.country_name value: '{{{aws.guardduty.service.action.aws_api_call_action.remote_ip_details.country.name}}}' if: ctx.aws?.guardduty?.service?.action?.aws_api_call_action?.remote_ip_details?.country?.name != null allow_duplicates: false - append: + tag: append_source_geo_country_name_1 field: source.geo.country_name value: '{{{aws.guardduty.service.action.kubernetes_api_call_action.remote_ip_details.country.name}}}' if: ctx.aws?.guardduty?.service?.action?.kubernetes_api_call_action?.remote_ip_details?.country?.name != null allow_duplicates: false - append: + tag: append_source_geo_country_name_2 field: source.geo.country_name value: '{{{aws.guardduty.service.action.network_connection_action.remote_ip_details.country.name}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.remote_ip_details?.country?.name != null allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_service_action_port_probe_action_port_probe_details_5 field: aws.guardduty.service.action.port_probe_action.port_probe_details if: ctx.aws?.guardduty?.service?.action?.port_probe_action?.port_probe_details instanceof List processor: @@ -2331,11 +2766,13 @@ processors: value: '{{{_ingest._value.remote_ip_details.country.name}}}' allow_duplicates: false - append: + tag: append_source_geo_country_name_3 field: source.geo.country_name value: '{{{aws.guardduty.service.action.rds_login_attempt_action.remote_ip_details.country.name}}}' if: ctx.aws?.guardduty?.service?.action?.rds_login_attempt_action?.remote_ip_details?.country?.name != null allow_duplicates: false - script: + tag: script description: Map source.geo.location field. lang: painless source: | @@ -2365,43 +2802,53 @@ processors: } ctx.source.geo.location = locationList; - set: + tag: set_dns_question_name field: dns.question.name copy_from: aws.guardduty.service.action.dns_request_action.domain ignore_empty_value: true - set: + tag: set_event_action field: event.action copy_from: aws.guardduty.service.action.type ignore_empty_value: true - set: + tag: set_event_created field: event.created copy_from: aws.guardduty.created_at ignore_empty_value: true - set: + tag: set_event_end field: event.end copy_from: aws.guardduty.service.event.last_seen ignore_empty_value: true - set: + tag: set_event_id field: event.id copy_from: aws.guardduty.id ignore_empty_value: true - set: + tag: set_event_provider field: event.provider copy_from: aws.guardduty.service.action.aws_api_call_action.service_name ignore_empty_value: true - convert: + tag: convert_json_severity_1 field: json.severity target_field: event.severity type: long ignore_missing: true on_failure: - append: + tag: append_error_message_62 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_event_start field: event.start copy_from: aws.guardduty.service.event.first_seen ignore_empty_value: true - foreach: + tag: foreach_aws_guardduty_service_ebs_volume_scan_details_scan_detections_threat_detected_by_name_threat_names field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.threat_detected_by_name.threat_names if: ctx.aws?.guardduty?.service?.ebs_volume_scan_details?.scan?.detections?.threat_detected_by_name?.threat_names instanceof List processor: @@ -2414,6 +2861,7 @@ processors: value: '{{{_ingest._value.hash}}}' allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_service_ebs_volume_scan_details_scan_detections_threat_detected_by_name_threat_names_1 field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.threat_detected_by_name.threat_names if: ctx.aws?.guardduty?.service?.ebs_volume_scan_details?.scan?.detections?.threat_detected_by_name?.threat_names instanceof List processor: @@ -2426,6 +2874,7 @@ processors: value: '{{{_ingest._value.file.name}}}' allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_service_ebs_volume_scan_details_scan_detections_threat_detected_by_name_threat_names_2 field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.threat_detected_by_name.threat_names if: ctx.aws?.guardduty?.service?.ebs_volume_scan_details?.scan?.detections?.threat_detected_by_name?.threat_names instanceof List processor: @@ -2438,83 +2887,103 @@ processors: value: '{{{_ingest._value.file.path}}}' allow_duplicates: false - set: + tag: set_cloud_instance_id field: cloud.instance.id copy_from: aws.guardduty.resource.instance_details.instance.id ignore_empty_value: true - set: + tag: set_host_id field: host.id copy_from: aws.guardduty.resource.instance_details.instance.id ignore_empty_value: true - set: + tag: set_host_os_platform field: host.os.platform copy_from: aws.guardduty.resource.instance_details.platform ignore_empty_value: true - set: + tag: set_cloud_machine_type field: cloud.machine.type copy_from: aws.guardduty.resource.instance_details.instance.type ignore_empty_value: true - set: + tag: set_host_type field: host.type copy_from: aws.guardduty.resource.instance_details.instance.type ignore_empty_value: true - set: + tag: set_network_iana_number field: network.iana_number copy_from: aws.guardduty.service.runtime_details.context.iana_protocol_number ignore_empty_value: true - lowercase: + tag: lowercase_aws_guardduty_service_action_network_connection_action_connection_direction field: aws.guardduty.service.action.network_connection_action.connection_direction target_field: network.direction ignore_missing: true - append: + tag: append_network_transport field: network.transport value: '{{{aws.guardduty.service.action.network_connection_action.transport}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.transport != null allow_duplicates: false - append: + tag: append_network_transport_1 field: network.transport value: '{{{aws.guardduty.service.action.dns_request_action.protocol}}}' if: ctx.aws?.guardduty?.service?.action?.dns_request_action?.protocol != null allow_duplicates: false - lowercase: + tag: lowercase_network_transport field: network.transport ignore_missing: true - set: + tag: set_orchestrator_namespace field: orchestrator.namespace copy_from: aws.guardduty.resource.kubernetes_details.kubernetes_workload_details.name_space ignore_empty_value: true - set: + tag: set_orchestrator_resource_name field: orchestrator.resource.name copy_from: aws.guardduty.resource.kubernetes_details.kubernetes_workload_details.name ignore_empty_value: true - set: + tag: set_orchestrator_resource_type field: orchestrator.resource.type copy_from: aws.guardduty.resource.kubernetes_details.kubernetes_workload_details.type ignore_empty_value: true - set: + tag: set_process_executable field: process.executable copy_from: aws.guardduty.service.runtime_details.process.executable_path ignore_empty_value: true - set: + tag: set_process_name field: process.name copy_from: aws.guardduty.service.runtime_details.process.name ignore_empty_value: true - set: + tag: set_process_pid field: process.pid copy_from: aws.guardduty.service.runtime_details.process.pid ignore_empty_value: true - set: + tag: set_process_start field: process.start copy_from: aws.guardduty.service.runtime_details.process.start_time ignore_empty_value: true - set: + tag: set_process_working_directory field: process.working_directory copy_from: aws.guardduty.service.runtime_details.process.pwd ignore_empty_value: true - set: + tag: set_rule_name field: rule.name copy_from: aws.guardduty.type ignore_empty_value: true - grok: + tag: grok_rule_name field: rule.name patterns: - '%{RULESET:rule.ruleset}' @@ -2522,6 +2991,7 @@ processors: RULESET: '%{WORD:rule.category}:%{WORD}' ignore_missing: true - foreach: + tag: foreach_aws_guardduty_service_action_port_probe_action_port_probe_details_6 field: aws.guardduty.service.action.port_probe_action.port_probe_details if: ctx.aws?.guardduty?.service?.action?.port_probe_action?.port_probe_details instanceof List processor: @@ -2530,6 +3000,7 @@ processors: value: '{{{_ingest._value.local_ip_details.ip_address_v4}}}' allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_service_action_kubernetes_api_call_action_source_ips field: aws.guardduty.service.action.kubernetes_api_call_action.source_ips if: ctx.aws?.guardduty?.service?.action?.kubernetes_api_call_action?.source_ips instanceof List processor: @@ -2538,11 +3009,13 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - append: + tag: append_source_port field: source.port value: '{{{aws.guardduty.service.action.network_connection_action.local_port_details.port.value}}}' if: ctx.aws?.guardduty?.service?.action?.network_connection_action?.local_port_details?.port?.value != null allow_duplicates: false - foreach: + tag: foreach_aws_guardduty_service_action_port_probe_action_port_probe_details_7 field: aws.guardduty.service.action.port_probe_action.port_probe_details if: ctx.aws?.guardduty?.service?.action?.port_probe_action?.port_probe_details instanceof List processor: @@ -2551,6 +3024,7 @@ processors: value: '{{{_ingest._value.local_port_details.port.value}}}' allow_duplicates: false - foreach: + tag: foreach_source_port field: source.port if: ctx.source?.port instanceof List processor: @@ -2559,46 +3033,56 @@ processors: type: long ignore_failure: true - set: + tag: set_threat_indicator_file_path field: threat.indicator.file.path copy_from: aws.guardduty.service.runtime_details.context.threat_file_path ignore_empty_value: true - set: + tag: set_threat_software_name field: threat.software.name copy_from: aws.guardduty.service.runtime_details.context.tool_name ignore_empty_value: true - append: + tag: append_user_id field: user.id value: '{{{aws.guardduty.resource.access_key_details.principal_id}}}' if: ctx.aws?.guardduty?.resource?.access_key_details?.principal_id != null allow_duplicates: false - append: + tag: append_user_id_1 field: user.id value: '{{{aws.guardduty.resource.kubernetes_details.kubernetes_user_details.uid}}}' if: ctx.aws?.guardduty?.resource?.kubernetes_details?.kubernetes_user_details?.uid != null allow_duplicates: false - append: + tag: append_user_name field: user.name value: '{{{aws.guardduty.resource.access_key_details.user.name}}}' if: ctx.aws?.guardduty?.resource?.access_key_details?.user?.name != null allow_duplicates: false - append: + tag: append_user_name_1 field: user.name value: '{{{aws.guardduty.resource.kubernetes_details.kubernetes_user_details.user_name}}}' if: ctx.aws?.guardduty?.resource?.kubernetes_details?.kubernetes_user_details?.user_name != null allow_duplicates: false - append: + tag: append_user_name_2 field: user.name value: '{{{aws.guardduty.resource.rdsdb_user_details.user}}}' if: ctx.aws?.guardduty?.resource?.rdsdb_user_details?.user != null allow_duplicates: false - set: + tag: set_user_roles field: user.roles copy_from: aws.guardduty.resource.kubernetes_details.kubernetes_user_details.groups ignore_empty_value: true - remove: + tag: remove_json field: json ignore_missing: true - remove: + tag: remove if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - aws.guardduty.account_id @@ -2671,6 +3155,7 @@ processors: - aws.guardduty.updated_at ignore_missing: true - foreach: + tag: foreach_aws_guardduty_resource_ecs_cluster_details_task_details_containers_3 field: aws.guardduty.resource.ecs_cluster_details.task_details.containers if: ctx.aws?.guardduty?.resource?.ecs_cluster_details?.task_details?.containers instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) processor: @@ -2681,6 +3166,7 @@ processors: - _ingest._value.name ignore_missing: true - foreach: + tag: foreach_aws_guardduty_service_action_port_probe_action_port_probe_details_8 field: aws.guardduty.service.action.port_probe_action.port_probe_details if: ctx.aws?.guardduty?.service?.action?.port_probe_action?.port_probe_details instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) processor: @@ -2697,6 +3183,7 @@ processors: - _ingest._value.remote_ip_details.organization.asnorg ignore_missing: true - foreach: + tag: foreach_aws_guardduty_service_ebs_volume_scan_details_scan_detections_threat_detected_by_name_threat_names_3 field: aws.guardduty.service.ebs_volume_scan_details.scan.detections.threat_detected_by_name.threat_names if: ctx.aws?.guardduty?.service?.ebs_volume_scan_details?.scan?.detections?.threat_detected_by_name?.threat_names instanceof List && (ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))) processor: @@ -2711,6 +3198,7 @@ processors: - _ingest._value.hash ignore_missing: true - script: + tag: script_1 description: Drops null/empty values recursively. lang: painless source: | @@ -2728,17 +3216,20 @@ processors: } dropEmptyFields(ctx); - append: + tag: append_event_kind field: event.kind value: pipeline_error if: ctx.error?.message != null allow_duplicates: false on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message_63 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/inspector/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/inspector/elasticsearch/ingest_pipeline/default.yml index 769eb4cf69e..9ed0d086138 100644 --- a/packages/aws/data_stream/inspector/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/inspector/elasticsearch/ingest_pipeline/default.yml @@ -66,6 +66,7 @@ processors: target_field: json on_failure: - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - fingerprint: @@ -85,6 +86,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -144,6 +146,7 @@ processors: } on_failure: - append: + tag: append_error_message_2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -188,6 +191,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -208,6 +212,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -233,6 +238,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -246,6 +252,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -300,6 +307,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -315,6 +323,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -335,6 +344,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -378,9 +388,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_10 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_networkreachabilitydetails_networkpath_steps field: json.networkReachabilityDetails.networkPath.steps if: ctx.json?.networkReachabilityDetails?.networkPath?.steps instanceof List processor: @@ -390,6 +402,7 @@ processors: target_field: _ingest._value.component.id ignore_missing: true - foreach: + tag: foreach_json_networkreachabilitydetails_networkpath_steps_1 field: json.networkReachabilityDetails.networkPath.steps if: ctx.json?.networkReachabilityDetails?.networkPath?.steps instanceof List processor: @@ -399,6 +412,7 @@ processors: target_field: _ingest._value.component.type ignore_missing: true - foreach: + tag: foreach_json_networkreachabilitydetails_networkpath_steps_2 field: json.networkReachabilityDetails.networkPath.steps if: ctx.json?.networkReachabilityDetails?.networkPath?.steps instanceof List processor: @@ -420,6 +434,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_11 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -430,9 +445,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_12 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_packagevulnerabilitydetails_cvss field: json.packageVulnerabilityDetails.cvss if: ctx.json?.packageVulnerabilityDetails?.cvss instanceof List processor: @@ -444,12 +461,15 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_basescore field: _ingest._value.baseScore ignore_missing: true - append: + tag: append_error_message_13 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_packagevulnerabilitydetails_cvss_1 field: json.packageVulnerabilityDetails.cvss if: ctx.json?.packageVulnerabilityDetails?.cvss instanceof List processor: @@ -458,6 +478,7 @@ processors: tag: remove_packageVulnerabilityDetails_cvss_baseScore ignore_missing: true - foreach: + tag: foreach_json_packagevulnerabilitydetails_cvss_2 field: json.packageVulnerabilityDetails.cvss if: ctx.json?.packageVulnerabilityDetails?.cvss instanceof List processor: @@ -473,7 +494,7 @@ processors: ignore_missing: true - rename: field: json.networkReachabilityDetails.networkPath.steps - tag: rename_networkReachabilityDetails_networkPath_steps + tag: rename_networkReachabilityDetails_networkPath_steps_1 target_field: aws.inspector.network_reachability_details.network_path.steps ignore_missing: true - rename: @@ -494,9 +515,11 @@ processors: keep_original: true on_failure: - remove: + tag: remove_json_packagevulnerabilitydetails_sourceurl field: json.packageVulnerabilityDetails.sourceUrl ignore_missing: true - append: + tag: append_error_message_14 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -510,6 +533,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_15 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -533,9 +557,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_16 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_packagevulnerabilitydetails_vulnerablepackages field: json.packageVulnerabilityDetails.vulnerablePackages if: ctx.json?.packageVulnerabilityDetails?.vulnerablePackages instanceof List processor: @@ -545,6 +571,7 @@ processors: target_field: _ingest._value.file_path ignore_missing: true - foreach: + tag: foreach_json_packagevulnerabilitydetails_vulnerablepackages_1 field: json.packageVulnerabilityDetails.vulnerablePackages if: ctx.json?.packageVulnerabilityDetails?.vulnerablePackages instanceof List processor: @@ -554,6 +581,7 @@ processors: target_field: _ingest._value.fixed_in_version ignore_missing: true - foreach: + tag: foreach_json_packagevulnerabilitydetails_vulnerablepackages_2 field: json.packageVulnerabilityDetails.vulnerablePackages if: ctx.json?.packageVulnerabilityDetails?.vulnerablePackages instanceof List processor: @@ -563,6 +591,7 @@ processors: target_field: _ingest._value.package_manager ignore_missing: true - foreach: + tag: foreach_json_packagevulnerabilitydetails_vulnerablepackages_3 field: json.packageVulnerabilityDetails.vulnerablePackages if: ctx.json?.packageVulnerabilityDetails?.vulnerablePackages instanceof List processor: @@ -572,6 +601,7 @@ processors: target_field: _ingest._value.source_lambda_layer_arn ignore_missing: true - foreach: + tag: foreach_json_packagevulnerabilitydetails_vulnerablepackages_4 field: json.packageVulnerabilityDetails.vulnerablePackages if: ctx.json?.packageVulnerabilityDetails?.vulnerablePackages instanceof List processor: @@ -581,6 +611,7 @@ processors: target_field: _ingest._value.source_layer_hash ignore_missing: true - foreach: + tag: foreach_json_packagevulnerabilitydetails_vulnerablepackages_5 field: json.packageVulnerabilityDetails.vulnerablePackages if: ctx.json?.packageVulnerabilityDetails?.vulnerablePackages instanceof List processor: @@ -601,6 +632,7 @@ processors: copy_from: aws.inspector.package_vulnerability_details.vulnerable_packages ignore_empty_value: true - foreach: + tag: foreach_aws_inspector_package_vulnerability_details_vulnerable_packages field: aws.inspector.package_vulnerability_details.vulnerable_packages if: ctx.aws?.inspector?.package_vulnerability_details?.vulnerable_packages instanceof List processor: @@ -610,6 +642,7 @@ processors: value: '{{{_ingest._value.arch}}}' allow_duplicates: false - foreach: + tag: foreach_aws_inspector_package_vulnerability_details_vulnerable_packages_1 field: aws.inspector.package_vulnerability_details.vulnerable_packages if: ctx.aws?.inspector?.package_vulnerability_details?.vulnerable_packages instanceof List processor: @@ -619,6 +652,7 @@ processors: value: '{{{_ingest._value.name}}}' allow_duplicates: false - foreach: + tag: foreach_aws_inspector_package_vulnerability_details_vulnerable_packages_2 field: aws.inspector.package_vulnerability_details.vulnerable_packages if: ctx.aws?.inspector?.package_vulnerability_details?.vulnerable_packages instanceof List processor: @@ -628,6 +662,7 @@ processors: value: '{{{_ingest._value.version}}}' allow_duplicates: false - foreach: + tag: foreach_aws_inspector_package_vulnerability_details_vulnerable_packages_3 field: aws.inspector.package_vulnerability_details.vulnerable_packages if: ctx.aws?.inspector?.package_vulnerability_details?.vulnerable_packages instanceof List processor: @@ -637,6 +672,7 @@ processors: value: '{{{_ingest._value.file_path}}}' allow_duplicates: false - foreach: + tag: foreach_aws_inspector_package_vulnerability_details_vulnerable_packages_4 field: aws.inspector.package_vulnerability_details.vulnerable_packages if: ctx.aws?.inspector?.package_vulnerability_details?.vulnerable_packages instanceof List processor: @@ -658,12 +694,15 @@ processors: keep_original: true on_failure: - remove: + tag: remove_json_remediation_recommendation_url field: json.remediation.recommendation.Url ignore_missing: true - append: + tag: append_error_message_17 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resources field: json.resources if: ctx.json?.resources instanceof List processor: @@ -673,6 +712,7 @@ processors: target_field: _ingest._value.details.aws.ec2_instance.iam_instance_profile_arn ignore_missing: true - foreach: + tag: foreach_json_resources_1 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -682,6 +722,7 @@ processors: target_field: _ingest._value.details.aws.ec2_instance.image_id ignore_missing: true - foreach: + tag: foreach_json_resources_2 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -691,6 +732,7 @@ processors: target_field: _ingest._value.details.aws.ec2_instance.ipv4_addresses ignore_missing: true - foreach: + tag: foreach_json_resources_3 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -704,11 +746,14 @@ processors: type: ip on_failure: - remove: + tag: remove_ingest_value field: _ingest._value - append: + tag: append_error_message_18 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resources_4 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -722,6 +767,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_json_resources_5 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -731,6 +777,7 @@ processors: target_field: _ingest._value.details.aws.ec2_instance.ipv6_addresses ignore_missing: true - foreach: + tag: foreach_json_resources_6 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -744,11 +791,14 @@ processors: type: ip on_failure: - remove: + tag: remove_ingest_value_1 field: _ingest._value - append: + tag: append_error_message_19 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resources_7 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -762,6 +812,7 @@ processors: value: '{{{_ingest._value}}}' allow_duplicates: false - foreach: + tag: foreach_json_resources_8 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -771,6 +822,7 @@ processors: target_field: _ingest._value.details.aws.ec2_instance.key_name ignore_missing: true - foreach: + tag: foreach_json_resources_9 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -784,9 +836,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - remove: + tag: remove_ingest_value_details_awsec2instance_launchedat field: _ingest._value.details.awsEc2Instance.launchedAt ignore_missing: true - foreach: + tag: foreach_json_resources_10 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -796,6 +850,7 @@ processors: target_field: _ingest._value.details.aws.ec2_instance.platform ignore_missing: true - foreach: + tag: foreach_json_resources_11 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -805,6 +860,7 @@ processors: target_field: _ingest._value.details.aws.ec2_instance.subnet_id ignore_missing: true - foreach: + tag: foreach_json_resources_12 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -814,6 +870,7 @@ processors: target_field: _ingest._value.details.aws.ec2_instance.type ignore_missing: true - foreach: + tag: foreach_json_resources_13 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -823,6 +880,7 @@ processors: target_field: _ingest._value.details.aws.ec2_instance.vpc_id ignore_missing: true - foreach: + tag: foreach_json_resources_14 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -832,6 +890,7 @@ processors: target_field: _ingest._value.details.aws.ecr_container_image.architecture ignore_missing: true - foreach: + tag: foreach_json_resources_15 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -841,6 +900,7 @@ processors: target_field: _ingest._value.details.aws.ecr_container_image.author ignore_missing: true - foreach: + tag: foreach_json_resources_16 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -850,6 +910,7 @@ processors: target_field: _ingest._value.details.aws.ecr_container_image.image.hash ignore_missing: true - foreach: + tag: foreach_json_resources_17 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -859,6 +920,7 @@ processors: value: '{{{_ingest._value.details.aws.ecr_container_image.image.hash}}}' allow_duplicates: false - foreach: + tag: foreach_json_resources_18 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -868,6 +930,7 @@ processors: target_field: _ingest._value.details.aws.ecr_container_image.image.tags ignore_missing: true - foreach: + tag: foreach_json_resources_19 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -879,12 +942,15 @@ processors: ignore_missing: true on_failure: - remove: + tag: remove_ingest_value_details_awsecrcontainerimage_inusecount field: _ingest._value.details.awsEcrContainerImage.inUseCount ignore_missing: true - append: + tag: append_error_message_20 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_resources_20 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -898,9 +964,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - remove: + tag: remove_ingest_value_details_awsecrcontainerimage_lastinuseat field: _ingest._value.details.awsEcrContainerImage.lastInUseAt ignore_missing: true - foreach: + tag: foreach_json_resources_21 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -910,6 +978,7 @@ processors: target_field: _ingest._value.details.aws.ecr_container_image.platform ignore_missing: true - foreach: + tag: foreach_json_resources_22 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -923,9 +992,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - remove: + tag: remove_ingest_value_details_awsecrcontainerimage_pushedat field: _ingest._value.details.awsEcrContainerImage.pushedAt ignore_missing: true - foreach: + tag: foreach_json_resources_23 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -935,6 +1006,7 @@ processors: target_field: _ingest._value.details.aws.ecr_container_image.registry ignore_missing: true - foreach: + tag: foreach_json_resources_24 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -944,6 +1016,7 @@ processors: target_field: _ingest._value.details.aws.ecr_container_image.repository_name ignore_missing: true - foreach: + tag: foreach_json_resources_25 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -953,6 +1026,7 @@ processors: target_field: _ingest._value.details.awsLambdaFunction.code_sha256 ignore_missing: true - foreach: + tag: foreach_json_resources_26 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -962,6 +1036,7 @@ processors: value: '{{{_ingest._value.details.awsLambdaFunction.code_sha256}}}' allow_duplicates: false - foreach: + tag: foreach_json_resources_27 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -971,6 +1046,7 @@ processors: target_field: _ingest._value.details.awsLambdaFunction.execution_role_arn ignore_missing: true - foreach: + tag: foreach_json_resources_28 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -980,6 +1056,7 @@ processors: target_field: _ingest._value.details.awsLambdaFunction.function_name ignore_missing: true - foreach: + tag: foreach_json_resources_29 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -993,9 +1070,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - remove: + tag: remove_ingest_value_details_awslambdafunction_lastmodifiedat field: _ingest._value.details.awsLambdaFunction.lastModifiedAt ignore_missing: true - foreach: + tag: foreach_json_resources_30 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -1005,6 +1084,7 @@ processors: target_field: _ingest._value.details.awsLambdaFunction.package_type ignore_missing: true - foreach: + tag: foreach_json_resources_31 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -1014,6 +1094,7 @@ processors: target_field: _ingest._value.details.awsLambdaFunction.vpc_config.security_group_ids ignore_missing: true - foreach: + tag: foreach_json_resources_32 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -1023,6 +1104,7 @@ processors: target_field: _ingest._value.details.awsLambdaFunction.vpc_config.subnet_ids ignore_missing: true - foreach: + tag: foreach_json_resources_33 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -1032,6 +1114,7 @@ processors: target_field: _ingest._value.details.awsLambdaFunction.vpc_config.vpc_id ignore_missing: true - foreach: + tag: foreach_json_resources_34 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -1045,6 +1128,7 @@ processors: tag: remove_resources_fields ignore_missing: true - foreach: + tag: foreach_json_resources_35 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -1054,6 +1138,7 @@ processors: target_field: _ingest._value.details.aws.lambda_function ignore_missing: true - foreach: + tag: foreach_json_resources_36 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -1063,6 +1148,7 @@ processors: target_field: _ingest._value.details.code_repository.integration_arn ignore_missing: true - foreach: + tag: foreach_json_resources_37 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -1072,6 +1158,7 @@ processors: target_field: _ingest._value.details.code_repository.project_name ignore_missing: true - foreach: + tag: foreach_json_resources_38 field: json.resources if: ctx.json?.resources instanceof List processor: @@ -1145,6 +1232,7 @@ processors: } on_failure: - append: + tag: append_error_message_21 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - script: @@ -1211,6 +1299,7 @@ processors: } on_failure: - append: + tag: append_error_message_22 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -1295,22 +1384,25 @@ processors: value: pipeline_error if: ctx.error?.message != null - append: + tag: append_tags field: tags value: preserve_original_event allow_duplicates: false if: ctx.error?.message != null on_failure: - append: + tag: append_error_message_23 field: error.message value: |- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' - set: field: event.kind tag: set_pipeline_error_to_event_kind value: pipeline_error - append: + tag: append_tags_1 field: tags value: preserve_original_event allow_duplicates: false diff --git a/packages/aws/data_stream/kafka_metrics/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/kafka_metrics/elasticsearch/ingest_pipeline/default.yml index d934623c84f..6c653e28f03 100644 --- a/packages/aws/data_stream/kafka_metrics/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/kafka_metrics/elasticsearch/ingest_pipeline/default.yml @@ -3,31 +3,38 @@ description: "Pipeline for Amazon MSK metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - rename: + tag: rename_aws_dimensions_cluster_name field: aws.dimensions.Cluster Name target_field: aws.dimensions.ClusterName ignore_missing: true - rename: + tag: rename_aws_dimensions_consumer_group field: aws.dimensions.Consumer Group target_field: aws.dimensions.ConsumerGroup ignore_missing: true - rename: + tag: rename_aws_dimensions_broker_id field: aws.dimensions.Broker ID target_field: aws.dimensions.BrokerID ignore_missing: true - rename: + tag: rename_aws_dimensions_client_authentication field: aws.dimensions.Client Authentication target_field: aws.dimensions.ClientAuthentication ignore_missing: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/kinesis/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/kinesis/elasticsearch/ingest_pipeline/default.yml index ed2097e9595..ec2f233c3ee 100644 --- a/packages/aws/data_stream/kinesis/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/kinesis/elasticsearch/ingest_pipeline/default.yml @@ -3,20 +3,24 @@ description: "Pipeline for Kinesis metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/lambda/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/lambda/elasticsearch/ingest_pipeline/default.yml index 4c66bc10291..77fa55b3d59 100644 --- a/packages/aws/data_stream/lambda/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/lambda/elasticsearch/ingest_pipeline/default.yml @@ -3,20 +3,24 @@ description: "Pipeline for Lambda metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/aws-lambda-json.yml b/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/aws-lambda-json.yml index 30deaa1f2ef..ace823aec7b 100644 --- a/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/aws-lambda-json.yml +++ b/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/aws-lambda-json.yml @@ -2,12 +2,14 @@ description: Pipeline for parsing AWS lambda logs in JSON format processors: - json: + tag: json_event_original field: event.original target_field: parsed ignore_failure: true # 1. Powertools structured logs (timestamp field) - date: + tag: date_parsed_timestamp if: "ctx.parsed?.timestamp != null" field: parsed.timestamp target_field: "@timestamp" @@ -16,6 +18,7 @@ processors: # 2. Platform report logs (time field) - date: + tag: date_parsed_time if: "ctx.parsed?.time != null" field: parsed.time target_field: "@timestamp" @@ -24,6 +27,7 @@ processors: # 3. EMF metrics (_aws.Timestamp) - date: + tag: date_parsed_aws_timestamp if: "ctx.parsed?._aws?.Timestamp != null" field: parsed._aws.Timestamp target_field: "@timestamp" @@ -32,33 +36,39 @@ processors: # Flatten important fields from each log type - set: + tag: set_aws_lambda_message field: aws.lambda.message if: "ctx.parsed?.record != null" copy_from: parsed.record ignore_failure: true - set: + tag: set_aws_lambda_message_1 field: aws.lambda.message if: "ctx.parsed?._aws != null" copy_from: parsed._aws ignore_failure: true - rename: + tag: rename_parsed_service field: parsed.service target_field: service.name ignore_missing: true - rename: + tag: rename_parsed_level field: parsed.level target_field: log.level ignore_missing: true - rename: + tag: rename_parsed_requestid field: parsed.requestId target_field: aws.lambda.request_id ignore_missing: true - script: + tag: script description: "Join stackTrace arrays into single multiline strings if present" if: "ctx.parsed instanceof Map" lang: painless @@ -87,24 +97,28 @@ processors: } - rename: + tag: rename_parsed_record_functionarn field: parsed.record.functionArn target_field: aws.lambda.arn ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_requestid field: parsed.record.requestId target_field: aws.lambda.request_id ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_version field: parsed.record.version target_field: aws.lambda.version ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_status field: parsed.record.status target_field: aws.lambda.status ignore_missing: true @@ -134,90 +148,105 @@ processors: # Rename Lambda tracing fields - rename: + tag: rename_parsed_record_tracing_spanid field: parsed.record.tracing.spanId target_field: aws.lambda.tracing.span_id ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_tracing_type field: parsed.record.tracing.type target_field: aws.lambda.tracing.type ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_tracing_value field: parsed.record.tracing.value target_field: aws.lambda.tracing.value ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_errortype field: parsed.record.errorType target_field: aws.lambda.error.type ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_initializationtype field: parsed.record.initializationType target_field: aws.lambda.initialization_type ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_phase field: parsed.record.phase target_field: aws.lambda.phase ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_functionversion field: parsed.record.functionVersion target_field: aws.lambda.version ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_functionname field: parsed.record.functionName target_field: aws.lambda.name ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_instanceid field: parsed.record.instanceId target_field: aws.lambda.instance_id ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_runtimeversion field: parsed.record.runtimeVersion target_field: aws.lambda.runtime_version ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_runtimeversionarn field: parsed.record.runtimeVersionArn target_field: aws.lambda.runtime_version_arn ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_name field: parsed.record.name target_field: aws.lambda.extension.name ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_state field: parsed.record.state target_field: aws.lambda.extension.state ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_events field: parsed.record.events target_field: aws.lambda.extension.events ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_record_spans field: parsed.record.spans target_field: aws.lambda.spans ignore_missing: true @@ -225,6 +254,7 @@ processors: # General fields - rename: + tag: rename_parsed_time field: parsed.time target_field: "@timestamp" ignore_missing: true @@ -232,84 +262,98 @@ processors: if: "ctx['@timestamp'] == null" - rename: + tag: rename_parsed_errormessage field: parsed.errorMessage target_field: aws.lambda.error.message ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_errortype field: parsed.errorType target_field: aws.lambda.error.type ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_logger field: parsed.logger target_field: log.logger ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_stack_trace_flattened field: parsed.stack_trace_flattened target_field: aws.lambda.error.stack_trace ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_message_stack_trace_flattened field: parsed.message.stack_trace_flattened target_field: aws.lambda.error.stack_trace ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_message_errortype field: parsed.message.errorType target_field: aws.lambda.error.type ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_message_errormessage field: parsed.message.errorMessage target_field: aws.lambda.error.message ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_time_1 field: parsed.time target_field: "@timestamp" ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_timestamp field: parsed.timestamp target_field: "@timestamp" ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_type field: parsed.type target_field: aws.lambda.event_type ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_cold_start field: parsed.cold_start target_field: aws.lambda.cold_start ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_correlation_id field: parsed.correlation_id target_field: aws.lambda.correlation_id ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_function_arn field: parsed.function_arn target_field: aws.lambda.arn ignore_missing: true ignore_failure: true - convert: + tag: convert_parsed_function_memory_size field: parsed.function_memory_size target_field: aws.lambda.metrics.memory_size_mb type: long @@ -317,96 +361,112 @@ processors: ignore_failure: true - rename: + tag: rename_parsed_function_name field: parsed.function_name target_field: aws.lambda.name ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_function_request_id field: parsed.function_request_id target_field: aws.lambda.request_id ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_location field: parsed.location target_field: aws.lambda.error.location ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_message field: parsed.message target_field: aws.lambda.message ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_users field: parsed.users target_field: aws.lambda.users ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_timestamp_1 field: parsed.timestamp target_field: "@timestamp" ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_tracing_xray_trace_id field: parsed.tracing.xray_trace_id target_field: aws.lambda.xray_trace_id ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_coldstart field: parsed.ColdStart target_field: aws.lambda.cold_start_int ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_functionname field: parsed.FunctionName target_field: aws.lambda.name ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_service_1 field: parsed.Service target_field: aws.lambda.service.name ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_executionenvironment field: parsed.executionEnvironment target_field: aws.lambda.execution_environment ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_functionversion field: parsed.functionVersion target_field: aws.lambda.version ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_logstreamid field: parsed.logStreamId target_field: aws.lambda.log_stream_id ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_traceid field: parsed.traceId target_field: aws.lambda.trace_id ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed_awsrequestid field: parsed.AWSRequestId target_field: aws.lambda.aws_request_id ignore_missing: true ignore_failure: true - rename: + tag: rename_parsed field: parsed target_field: aws.lambda.message ignore_missing: true @@ -414,21 +474,25 @@ processors: if: "ctx.parsed instanceof Map && !(ctx.parsed.containsKey('message') || ctx.parsed.containsKey('record') || ctx.parsed.containsKey('_aws') || ctx.parsed.containsKey('time') || ctx.parsed.containsKey('timestamp'))" - set: + tag: set_aws_lambda_message_2 field: aws.lambda.message copy_from: event.original if: "ctx.parsed == null" - remove: + tag: remove_parsed field: parsed ignore_missing: true ignore_failure: true - remove: + tag: remove_parsed_aws field: parsed._aws ignore_missing: true ignore_failure: true - script: + tag: script_1 description: Drops null/empty values recursively lang: painless source: | @@ -448,11 +512,13 @@ processors: on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - set: + tag: set_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' \ No newline at end of file + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' \ No newline at end of file diff --git a/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/aws-lambda-plaintext.yml b/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/aws-lambda-plaintext.yml index 3e693253042..8cc40bafe71 100644 --- a/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/aws-lambda-plaintext.yml +++ b/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/aws-lambda-plaintext.yml @@ -2,6 +2,7 @@ description: "Parse AWS Lambda logs with fallback to raw message" processors: - grok: + tag: grok_event_original field: event.original pattern_definitions: GREEDYMULTILINE: "(.|\n|\t)*" @@ -27,6 +28,7 @@ processors: ignore_failure: true - date: + tag: date_timestamp if: ctx.timestamp != null field: timestamp target_field: "@timestamp" @@ -38,22 +40,26 @@ processors: ignore_failure: true - set: + tag: set_timestamp if: "ctx['@timestamp'] == null" field: "@timestamp" value: "{{_ingest.timestamp}}" ignore_failure: true - remove: + tag: remove_timestamp field: timestamp ignore_missing: true ignore_failure: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - set: + tag: set_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' \ No newline at end of file + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' \ No newline at end of file diff --git a/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/default.yml index 6befc438239..af7c927093b 100644 --- a/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/lambda_logs/elasticsearch/ingest_pipeline/default.yml @@ -2,47 +2,58 @@ description: Pipeline for AWS Lambda logs. processors: - rename: + tag: rename_message field: message target_field: event.original if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: ctx.event?.original != null description: 'The `message` field is no longer required if the document has an `event.original` field.' - set: + tag: set_ecs_version field: ecs.version value: 8.11.0 - set: + tag: set_cloud_service_name field: cloud.service.name value: aws_lambda - set: + tag: set_cloud_provider field: cloud.provider value: aws - grok: + tag: grok_event_original field: event.original patterns: - '^%{CHAR:first_char}' pattern_definitions: CHAR: . - pipeline: + tag: pipeline if: ctx.first_char != '{' name: '{{ IngestPipeline "aws-lambda-plaintext" }}' - pipeline: + tag: pipeline_1 if: ctx.first_char == '{' name: '{{ IngestPipeline "aws-lambda-json" }}' - remove: + tag: remove_first_char field: first_char ignore_missing: true description: 'Removes the `first_char` field used to determine the log format.' on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' \ No newline at end of file + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' \ No newline at end of file diff --git a/packages/aws/data_stream/natgateway/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/natgateway/elasticsearch/ingest_pipeline/default.yml index efc2e68a46d..59c6694c524 100644 --- a/packages/aws/data_stream/natgateway/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/natgateway/elasticsearch/ingest_pipeline/default.yml @@ -3,20 +3,24 @@ description: "Pipeline for NAT Gateway metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/rds/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/rds/elasticsearch/ingest_pipeline/default.yml index 59404558f11..f0778b02f38 100644 --- a/packages/aws/data_stream/rds/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/rds/elasticsearch/ingest_pipeline/default.yml @@ -3,9 +3,11 @@ description: "Pipeline for RDS metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - script: + tag: script lang: painless description: This script converts aws.rds.metrics.CPUUtilization.avg from percentage to decimal. source: | @@ -13,288 +15,359 @@ processors: ctx.aws.rds.metrics.CPUUtilization.avg = ctx.aws.rds.metrics.CPUUtilization.avg / 100; } - rename: + tag: rename_aws_rds_metrics_burstbalance_avg field: aws.rds.metrics.BurstBalance.avg target_field: aws.rds.burst_balance.pct ignore_missing: true - rename: + tag: rename_aws_rds_metrics_cpuutilization_avg field: aws.rds.metrics.CPUUtilization.avg target_field: aws.rds.cpu.total.pct ignore_missing: true - rename: + tag: rename_aws_rds_metrics_cpucreditusage_avg field: aws.rds.metrics.CPUCreditUsage.avg target_field: aws.rds.cpu.credit_usage ignore_missing: true - rename: + tag: rename_aws_rds_metrics_cpucreditbalance_avg field: aws.rds.metrics.CPUCreditBalance.avg target_field: aws.rds.cpu.credit_balance ignore_missing: true - rename: + tag: rename_aws_rds_metrics_databaseconnections_avg field: aws.rds.metrics.DatabaseConnections.avg target_field: aws.rds.database_connections ignore_missing: true - rename: + tag: rename_aws_rds_metrics_diskqueuedepth_avg field: aws.rds.metrics.DiskQueueDepth.avg target_field: aws.rds.disk_queue_depth ignore_missing: true - rename: + tag: rename_aws_rds_metrics_failedsqlserveragentjobscount_avg field: aws.rds.metrics.FailedSQLServerAgentJobsCount.avg target_field: aws.rds.failed_sql_server_agent_jobs ignore_missing: true - rename: + tag: rename_aws_rds_metrics_freeablememory_avg field: aws.rds.metrics.FreeableMemory.avg target_field: aws.rds.freeable_memory.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_freestoragespace_avg field: aws.rds.metrics.FreeStorageSpace.avg target_field: aws.rds.free_storage.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_maximumusedtransactionids_avg field: aws.rds.metrics.MaximumUsedTransactionIDs.avg target_field: aws.rds.maximum_used_transaction_ids ignore_missing: true - rename: + tag: rename_aws_rds_metrics_oldestreplicationslotlag_avg field: aws.rds.metrics.OldestReplicationSlotLag.avg target_field: aws.rds.oldest_replication_slot_lag.mb ignore_missing: true - rename: + tag: rename_aws_rds_metrics_readiops_avg field: aws.rds.metrics.ReadIOPS.avg target_field: aws.rds.read.iops ignore_missing: true - rename: + tag: rename_aws_rds_metrics_committhroughput_avg field: aws.rds.metrics.CommitThroughput.avg target_field: aws.rds.throughput.commit ignore_missing: true - rename: + tag: rename_aws_rds_metrics_deletethroughput_avg field: aws.rds.metrics.DeleteThroughput.avg target_field: aws.rds.throughput.delete ignore_missing: true - rename: + tag: rename_aws_rds_metrics_ddlthroughput_avg field: aws.rds.metrics.DDLThroughput.avg target_field: aws.rds.throughput.ddl ignore_missing: true - rename: + tag: rename_aws_rds_metrics_dmlthroughput_avg field: aws.rds.metrics.DMLThroughput.avg target_field: aws.rds.throughput.dml ignore_missing: true - rename: + tag: rename_aws_rds_metrics_insertthroughput_avg field: aws.rds.metrics.InsertThroughput.avg target_field: aws.rds.throughput.insert ignore_missing: true - rename: + tag: rename_aws_rds_metrics_networkthroughput_avg field: aws.rds.metrics.NetworkThroughput.avg target_field: aws.rds.throughput.network ignore_missing: true - rename: + tag: rename_aws_rds_metrics_networkreceivethroughput_avg field: aws.rds.metrics.NetworkReceiveThroughput.avg target_field: aws.rds.throughput.network_receive ignore_missing: true - rename: + tag: rename_aws_rds_metrics_networktransmitthroughput_avg field: aws.rds.metrics.NetworkTransmitThroughput.avg target_field: aws.rds.throughput.network_transmit ignore_missing: true - rename: + tag: rename_aws_rds_metrics_readthroughput_avg field: aws.rds.metrics.ReadThroughput.avg target_field: aws.rds.throughput.read ignore_missing: true - rename: + tag: rename_aws_rds_metrics_selectthroughput_avg field: aws.rds.metrics.SelectThroughput.avg target_field: aws.rds.throughput.select ignore_missing: true - rename: + tag: rename_aws_rds_metrics_updatethroughput_avg field: aws.rds.metrics.UpdateThroughput.avg target_field: aws.rds.throughput.update ignore_missing: true - rename: + tag: rename_aws_rds_metrics_freestoragespace_avg_1 field: aws.rds.metrics.FreeStorageSpace.avg target_field: aws.rds.free_storage.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_writethroughput_avg field: aws.rds.metrics.WriteThroughput.avg target_field: aws.rds.throughput.write ignore_missing: true - rename: + tag: rename_aws_rds_metrics_commitlatency_avg field: aws.rds.metrics.CommitLatency.avg target_field: aws.rds.latency.commit ignore_missing: true - rename: + tag: rename_aws_rds_metrics_ddllatency_avg field: aws.rds.metrics.DDLLatency.avg target_field: aws.rds.latency.ddl ignore_missing: true - rename: + tag: rename_aws_rds_metrics_dmllatency_avg field: aws.rds.metrics.DMLLatency.avg target_field: aws.rds.latency.dml ignore_missing: true - rename: + tag: rename_aws_rds_metrics_insertlatency_avg field: aws.rds.metrics.InsertLatency.avg target_field: aws.rds.latency.insert ignore_missing: true - rename: + tag: rename_aws_rds_metrics_readlatency_avg field: aws.rds.metrics.ReadLatency.avg target_field: aws.rds.latency.read ignore_missing: true - rename: + tag: rename_aws_rds_metrics_selectlatency_avg field: aws.rds.metrics.SelectLatency.avg target_field: aws.rds.latency.select ignore_missing: true - rename: + tag: rename_aws_rds_metrics_updatelatency_avg field: aws.rds.metrics.UpdateLatency.avg target_field: aws.rds.latency.update ignore_missing: true - rename: + tag: rename_aws_rds_metrics_writelatency_avg field: aws.rds.metrics.WriteLatency.avg target_field: aws.rds.latency.write ignore_missing: true - rename: + tag: rename_aws_rds_metrics_deletelatency_avg field: aws.rds.metrics.DeleteLatency.avg target_field: aws.rds.latency.delete ignore_missing: true - rename: + tag: rename_aws_rds_metrics_replicalag_avg field: aws.rds.metrics.ReplicaLag.avg target_field: aws.rds.replica_lag.sec ignore_missing: true - rename: + tag: rename_aws_rds_metrics_binlogdiskusage_avg field: aws.rds.metrics.BinLogDiskUsage.avg target_field: aws.rds.disk_usage.bin_log.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_replicationslotdiskusage_avg field: aws.rds.metrics.ReplicationSlotDiskUsage.avg target_field: aws.rds.disk_usage.replication_slot.mb ignore_missing: true - rename: + tag: rename_aws_rds_metrics_transactionlogsdiskusage_avg field: aws.rds.metrics.TransactionLogsDiskUsage.avg target_field: aws.rds.disk_usage.transaction_logs.mb ignore_missing: true - rename: + tag: rename_aws_rds_metrics_swapusage_avg field: aws.rds.metrics.SwapUsage.avg target_field: aws.rds.swap_usage.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_transactionlogsgeneration_avg field: aws.rds.metrics.TransactionLogsGeneration.avg target_field: aws.rds.transaction_logs_generation ignore_missing: true - rename: + tag: rename_aws_rds_metrics_writeiops_avg field: aws.rds.metrics.WriteIOPS.avg target_field: aws.rds.write.iops ignore_missing: true - rename: + tag: rename_aws_rds_metrics_queries_avg field: aws.rds.metrics.Queries.avg target_field: aws.rds.queries ignore_missing: true - rename: + tag: rename_aws_rds_metrics_deadlocks_avg field: aws.rds.metrics.Deadlocks.avg target_field: aws.rds.deadlocks ignore_missing: true - rename: + tag: rename_aws_rds_metrics_volumebytesused_avg field: aws.rds.metrics.VolumeBytesUsed.avg target_field: aws.rds.volume_used.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_freelocalstorage_avg field: aws.rds.metrics.FreeLocalStorage.avg target_field: aws.rds.free_local_storage.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_activetransactions_avg field: aws.rds.metrics.ActiveTransactions.avg target_field: aws.rds.transactions.active ignore_missing: true - rename: + tag: rename_aws_rds_metrics_blockedtransactions_avg field: aws.rds.metrics.BlockedTransactions.avg target_field: aws.rds.transactions.blocked ignore_missing: true - rename: + tag: rename_aws_rds_metrics_loginfailures_avg field: aws.rds.metrics.LoginFailures.avg target_field: aws.rds.login_failures ignore_missing: true - rename: + tag: rename_aws_rds_metrics_aurorabinlogreplicalag_avg field: aws.rds.metrics.AuroraBinlogReplicaLag.avg target_field: aws.rds.aurora_bin_log_replica_lag ignore_missing: true - rename: + tag: rename_aws_rds_metrics_aurora_bin_log_replica_lag_avg field: aws.rds.metrics.aurora_bin_log_replica_lag.avg target_field: aws.rds.aurora_global_db.replicated_write_io.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_auroraglobaldbdatatransferbytes_avg field: aws.rds.metrics.AuroraGlobalDBDataTransferBytes.avg target_field: aws.rds.aurora_global_db.data_transfer.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_auroraglobaldbreplicationlag_avg field: aws.rds.metrics.AuroraGlobalDBReplicationLag.avg target_field: aws.rds.aurora_global_db.replication_lag.ms ignore_missing: true - rename: + tag: rename_aws_rds_metrics_aurorareplicalag_avg field: aws.rds.metrics.AuroraReplicaLag.avg target_field: aws.rds.aurora_replica.lag.ms ignore_missing: true - rename: + tag: rename_aws_rds_metrics_aurorareplicalagmaximum_avg field: aws.rds.metrics.AuroraReplicaLagMaximum.avg target_field: aws.rds.aurora_replica.lag_max.ms ignore_missing: true - rename: + tag: rename_aws_rds_metrics_aurorareplicalagminimum_avg field: aws.rds.metrics.AuroraReplicaLagMinimum.avg target_field: aws.rds.aurora_replica.lag_min.ms ignore_missing: true - rename: + tag: rename_aws_rds_metrics_backtrackchangerecordscreationrate_avg field: aws.rds.metrics.BacktrackChangeRecordsCreationRate.avg target_field: aws.rds.backtrack_change_records.creation_rate ignore_missing: true - rename: + tag: rename_aws_rds_metrics_backtrackchangerecordsstored_avg field: aws.rds.metrics.BacktrackChangeRecordsStored.avg target_field: aws.rds.backtrack_change_records.stored ignore_missing: true - rename: + tag: rename_aws_rds_metrics_backtrackwindowactual_avg field: aws.rds.metrics.BacktrackWindowActual.avg target_field: aws.rds.backtrack_window.actual ignore_missing: true - rename: + tag: rename_aws_rds_metrics_backtrackwindowalert_avg field: aws.rds.metrics.BacktrackWindowAlert.avg target_field: aws.rds.backtrack_window.alert ignore_missing: true - rename: + tag: rename_aws_rds_metrics_backupretentionperiodstorageused_avg field: aws.rds.metrics.BackupRetentionPeriodStorageUsed.avg target_field: aws.rds.storage_used.backup_retention_period.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_snapshotstorageused_avg field: aws.rds.metrics.SnapshotStorageUsed.avg target_field: aws.rds.storage_used.snapshot.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_buffercachehitratio_avg field: aws.rds.metrics.BufferCacheHitRatio.avg target_field: aws.rds.cache_hit_ratio.buffer ignore_missing: true - rename: + tag: rename_aws_rds_metrics_resultsetcachehitratio_avg field: aws.rds.metrics.ResultSetCacheHitRatio.avg target_field: aws.rds.cache_hit_ratio.result_set ignore_missing: true - rename: + tag: rename_aws_rds_metrics_engineuptime_avg field: aws.rds.metrics.EngineUptime.avg target_field: aws.rds.engine_uptime.sec ignore_missing: true - rename: + tag: rename_aws_rds_metrics_volumereadiops_avg field: aws.rds.metrics.VolumeReadIOPs.avg target_field: aws.rds.volume.read.iops ignore_missing: true - rename: + tag: rename_aws_rds_metrics_volumewriteiops_avg field: aws.rds.metrics.VolumeWriteIOPs.avg target_field: aws.rds.volume.write.iops ignore_missing: true - rename: + tag: rename_aws_rds_metrics_rdstoaurorapostgresqlreplicalag_avg field: aws.rds.metrics.RDSToAuroraPostgreSQLReplicaLag.avg target_field: aws.rds.rds_to_aurora_postgresql_replica_lag.sec ignore_missing: true - rename: + tag: rename_aws_rds_metrics_totalbackupstoragebilled_avg field: aws.rds.metrics.TotalBackupStorageBilled.avg target_field: aws.rds.backup_storage_billed_total.bytes ignore_missing: true - rename: + tag: rename_aws_rds_metrics_auroravolumebyteslefttotal_avg field: aws.rds.metrics.AuroraVolumeBytesLeftTotal.avg target_field: aws.rds.aurora_volume_left_total.bytes ignore_missing: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/redshift/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/redshift/elasticsearch/ingest_pipeline/default.yml index 6674abea2f7..1c7d7e99523 100644 --- a/packages/aws/data_stream/redshift/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/redshift/elasticsearch/ingest_pipeline/default.yml @@ -3,28 +3,34 @@ description: "Ingest Pipeline for Amazon Redshift metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true - rename: + tag: rename_aws_dimensions_service_class field: aws.dimensions.service class target_field: aws.dimensions.service_class ignore_missing: true - remove: + tag: remove field: - aws.dimensions.service class ignore_missing: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml index 717a2616624..82d49a552c2 100644 --- a/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,32 +3,40 @@ description: Pipeline for AWS Route53 Logs processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - set: + tag: set_cloud_provider field: cloud.provider value: aws - set: + tag: set_event_kind field: event.kind value: event - append: + tag: append_event_category field: event.category value: network - append: + tag: append_event_type field: event.type value: protocol - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - grok: + tag: grok_event_original field: event.original patterns: - '%{BASE10NUM} %{TIMESTAMP_ISO8601:_tmp.timestamp} %{DATA:aws.route53.hosted_zone_id} %{DATA:_tmp.question} %{WORD:dns.question.type} %{WORD:dns.response_code} %{WORD:network.transport} %{EDGE_LOCATION:aws.route53.edge_location} %{IP:source.address} (%{SUBNET:aws.route53.edns_client_subnet}|-)' @@ -36,62 +44,76 @@ processors: EDGE_LOCATION: '[A-Z]{3}\d+(-[A-Z]+\d+)?' SUBNET: '%{IP}/[0-9]+' - date: + tag: date_tmp_timestamp field: _tmp.timestamp target_field: '@timestamp' ignore_failure: true formats: - ISO8601 - set: + tag: set_event_outcome field: event.outcome value: success if: ctx.dns?.response_code == "NOERROR" - set: + tag: set_event_outcome_1 field: event.outcome value: failure if: ctx.dns?.response_code != "NOERROR" - registered_domain: + tag: registered_domain_tmp_question field: _tmp.question target_field: dns.question ignore_missing: true if: '!ctx._tmp?.question.endsWith("in-addr.arpa")' - rename: + tag: rename_dns_question_domain field: dns.question.domain target_field: dns.question.name ignore_missing: true - convert: + tag: convert_source_address field: source.address target_field: source.ip type: ip ignore_missing: true - lowercase: + tag: lowercase_network_transport field: network.transport ignore_missing: true - set: + tag: set_network_protocol field: network.protocol value: dns - set: + tag: set_network_type field: network.type value: ipv4 if: 'ctx.source?.ip != null && ctx.source?.ip.contains(".")' - set: + tag: set_network_type_1 field: network.type value: ipv6 if: 'ctx.source?.ip != null && ctx.source?.ip.contains(":")' - set: + tag: set_network_iana_number field: network.iana_number value: '6' if: ctx.network?.transport == "tcp" - set: + tag: set_network_iana_number_1 field: network.iana_number value: '17' if: ctx.network?.transport == "udp" # IP Geolocation Lookup - geoip: + tag: geoip_source_ip field: source.ip target_field: source.geo ignore_missing: true # IP Autonomous System (AS) Lookup - geoip: + tag: geoip_source_ip_1 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -100,32 +122,39 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - append: + tag: append_related_ip field: related.ip value: "{{source.ip}}" if: ctx.source?.ip != null - append: + tag: append_related_hosts field: related.hosts value: "{{dns.question.name}}" if: ctx.dns?.question?.name != null - remove: + tag: remove field: - _tmp ignore_missing: true on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml index 694b63b659a..4afb717a97c 100644 --- a/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,87 +3,107 @@ description: Pipeline for AWS Route53 Resolver Logs processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original field: event.original target_field: json - set: + tag: set_cloud_provider field: cloud.provider value: aws - rename: + tag: rename_json_account_id field: json.account_id target_field: cloud.account.id ignore_missing: true - date: + tag: date_json_query_timestamp field: json.query_timestamp target_field: '@timestamp' ignore_failure: true formats: - ISO8601 - set: + tag: set_cloud_region field: cloud.region copy_from: json.region ignore_empty_value: true - rename: + tag: rename_json_vpc_id field: json.vpc_id target_field: aws.vpc_id ignore_missing: true - rename: + tag: rename_json_srcids_instance field: json.srcids.instance target_field: aws.instance_id ignore_missing: true - set: + tag: set_cloud_instance_id field: cloud.instance.id copy_from: aws.instance_id ignore_empty_value: true - gsub: + tag: gsub_json_query_name field: json.query_name pattern: \.$ replacement: "" ignore_missing: true - registered_domain: + tag: registered_domain_json_query_name field: json.query_name target_field: dns.question ignore_missing: true if: '!ctx.json?.query_name.endsWith("in-addr.arpa") && !ctx.json?.query_name.endsWith("ip6.arpa")' - rename: + tag: rename_dns_question_domain field: dns.question.domain target_field: dns.question.name ignore_missing: true - rename: + tag: rename_json_query_name field: json.query_name target_field: dns.question.name ignore_missing: true if: ctx.dns?.question?.name == null - rename: + tag: rename_json_query_class field: json.query_class target_field: dns.question.class ignore_missing: true - rename: + tag: rename_json_query_type field: json.query_type target_field: dns.question.type ignore_missing: true - rename: + tag: rename_json_rcode field: json.rcode target_field: dns.response_code ignore_missing: true - rename: + tag: rename_json_answers field: json.answers target_field: dns.answers ignore_missing: true - script: + tag: script lang: painless ignore_failure: true if: ctx.dns?.answers != null && ctx.dns?.answers instanceof List @@ -125,44 +145,54 @@ processors: } ctx.dns.answers = answers; - rename: + tag: rename_json_transport field: json.transport target_field: network.transport ignore_missing: true - lowercase: + tag: lowercase_network_transport field: network.transport ignore_missing: true - set: + tag: set_network_iana_number field: network.iana_number value: '6' if: ctx.network?.transport == "tcp" - set: + tag: set_network_iana_number_1 field: network.iana_number value: '17' if: ctx.network?.transport == "udp" - set: + tag: set_network_protocol field: network.protocol value: dns - convert: + tag: convert_json_srcport field: json.srcport target_field: source.port type: long ignore_missing: true - rename: + tag: rename_json_srcaddr field: json.srcaddr target_field: source.address ignore_missing: true - convert: + tag: convert_source_address field: source.address target_field: source.ip type: ip ignore_missing: true # IP Geolocation Lookup - geoip: + tag: geoip_source_ip field: source.ip target_field: source.geo ignore_missing: true # IP Autonomous System (AS) Lookup - geoip: + tag: geoip_source_ip_1 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -171,55 +201,69 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - set: + tag: set_network_type field: network.type value: ipv4 if: 'ctx.source?.ip != null && ctx.source?.ip.contains(".")' - set: + tag: set_network_type_1 field: network.type value: ipv6 if: 'ctx.source?.ip != null && ctx.source?.ip.contains(":")' - rename: + tag: rename_json_firewall_rule_action field: json.firewall_rule_action target_field: aws.route53.firewall.action ignore_missing: true - rename: + tag: rename_json_firewall_rule_group_id field: json.firewall_rule_group_id target_field: aws.route53.firewall.rule_group.id ignore_missing: true - rename: + tag: rename_json_firewall_domain_list_id field: json.firewall_domain_list_id target_field: aws.route53.firewall.domain_list.id ignore_missing: true - set: + tag: set_event_kind field: event.kind value: event - append: + tag: append_event_category field: event.category value: network - append: + tag: append_event_type field: event.type value: protocol - set: + tag: set_event_outcome field: event.outcome value: success if: ctx.dns?.response_code == "NOERROR" - set: + tag: set_event_outcome_1 field: event.outcome value: failure if: ctx.dns?.response_code != "NOERROR" - append: + tag: append_related_ip field: related.ip value: "{{source.ip}}" if: ctx.source?.ip != null - script: + tag: script_1 lang: painless ignore_failure: true if: ctx.dns?.question?.name != null && ctx.dns?.question?.type == "PTR" @@ -253,14 +297,17 @@ processors: ctx.related.ip.add(ip); } - append: + tag: append_related_hosts field: related.hosts value: "{{dns.question.name}}" if: ctx.dns?.question?.name != null && ctx.dns?.question?.type != "PTR" - remove: + tag: remove field: - json ignore_missing: true - script: + tag: script_2 lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | @@ -287,11 +334,13 @@ processors: handleMap(ctx); on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/s3_daily_storage/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/s3_daily_storage/elasticsearch/ingest_pipeline/default.yml index 3a2ef129059..01f29eb874c 100644 --- a/packages/aws/data_stream/s3_daily_storage/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/s3_daily_storage/elasticsearch/ingest_pipeline/default.yml @@ -3,37 +3,45 @@ description: "Pipeline for S3 daily storage metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true - rename: + tag: rename_aws_s3_metrics_numberofobjects_avg field: aws.s3.metrics.NumberOfObjects.avg target_field: aws.s3_daily_storage.number_of_objects ignore_missing: true - rename: + tag: rename_aws_s3_metrics_bucketsizebytes_avg field: aws.s3.metrics.BucketSizeBytes.avg target_field: aws.s3_daily_storage.bucket.size.bytes ignore_missing: true - rename: + tag: rename_aws_dimensions_bucketname field: aws.dimensions.BucketName target_field: aws.s3.bucket.name ignore_missing: true - remove: + tag: remove field: - aws.s3.metrics if: ctx.agent?.type != "firehose" ignore_missing: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/s3_request/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/s3_request/elasticsearch/ingest_pipeline/default.yml index 0738d37c02a..b358d69441b 100644 --- a/packages/aws/data_stream/s3_request/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/s3_request/elasticsearch/ingest_pipeline/default.yml @@ -3,101 +3,125 @@ description: "Pipeline for S3 request metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true - rename: + tag: rename_aws_s3_metrics_allrequests_sum field: aws.s3.metrics.AllRequests.sum target_field: aws.s3_request.requests.total ignore_missing: true - rename: + tag: rename_aws_s3_metrics_getrequests_sum field: aws.s3.metrics.GetRequests.sum target_field: aws.s3_request.requests.get ignore_missing: true - rename: + tag: rename_aws_s3_metrics_putrequests_sum field: aws.s3.metrics.PutRequests.sum target_field: aws.s3_request.requests.put ignore_missing: true - rename: + tag: rename_aws_s3_metrics_deleterequests_sum field: aws.s3.metrics.DeleteRequests.sum target_field: aws.s3_request.requests.delete ignore_missing: true - rename: + tag: rename_aws_s3_metrics_headrequests_sum field: aws.s3.metrics.HeadRequests.sum target_field: aws.s3_request.requests.head ignore_missing: true - rename: + tag: rename_aws_s3_metrics_postrequests_sum field: aws.s3.metrics.PostRequests.sum target_field: aws.s3_request.requests.post ignore_missing: true - rename: + tag: rename_aws_s3_metrics_selectrequests_sum field: aws.s3.metrics.SelectRequests.sum target_field: aws.s3_request.requests.select ignore_missing: true - rename: + tag: rename_aws_s3_metrics_selectscannedbytes_avg field: aws.s3.metrics.SelectScannedBytes.avg target_field: aws.s3_request.requests.select_scanned.bytes ignore_missing: true - rename: + tag: rename_aws_s3_metrics_selectreturnedbytes_avg field: aws.s3.metrics.SelectReturnedBytes.avg target_field: aws.s3_request.requests.select_returned.bytes ignore_missing: true - rename: + tag: rename_aws_s3_metrics_listrequests_sum field: aws.s3.metrics.ListRequests.sum target_field: aws.s3_request.requests.list ignore_missing: true - rename: + tag: rename_aws_s3_metrics_bytesdownloaded_avg field: aws.s3.metrics.BytesDownloaded.avg target_field: aws.s3_request.downloaded.bytes ignore_missing: true - rename: + tag: rename_aws_s3_metrics_bytesuploaded_avg field: aws.s3.metrics.BytesUploaded.avg target_field: aws.s3_request.uploaded.bytes ignore_missing: true - rename: + tag: rename_aws_s3_metrics_bytesdownloaded_sum field: aws.s3.metrics.BytesDownloaded.sum target_field: aws.s3_request.downloaded.bytes_per_period ignore_missing: true - rename: + tag: rename_aws_s3_metrics_bytesuploaded_sum field: aws.s3.metrics.BytesUploaded.sum target_field: aws.s3_request.uploaded.bytes_per_period ignore_missing: true - rename: + tag: rename_aws_s3_metrics_4xxerrors_avg field: aws.s3.metrics.4xxErrors.avg target_field: aws.s3_request.errors.4xx ignore_missing: true - rename: + tag: rename_aws_s3_metrics_5xxerrors_avg field: aws.s3.metrics.5xxErrors.avg target_field: aws.s3_request.errors.5xx ignore_missing: true - rename: + tag: rename_aws_s3_metrics_firstbytelatency_avg field: aws.s3.metrics.FirstByteLatency.avg target_field: aws.s3_request.latency.first_byte.ms ignore_missing: true - rename: + tag: rename_aws_s3_metrics_totalrequestlatency_avg field: aws.s3.metrics.TotalRequestLatency.avg target_field: aws.s3_request.latency.total_request.ms ignore_missing: true - rename: + tag: rename_aws_dimensions_bucketname field: aws.dimensions.BucketName target_field: aws.s3.bucket.name ignore_missing: true - remove: + tag: remove field: - aws.s3.metrics if: ctx.agent?.type != "firehose" ignore_missing: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml index 426f98fe296..b1581cbd8e3 100644 --- a/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml @@ -3,26 +3,32 @@ description: "Pipeline for S3 server access logs" processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - set: + tag: set_event_category field: event.category value: ["web"] - append: + tag: append_event_type field: event.type value: ["access"] - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - grok: + tag: grok_event_original field: event.original patterns: - >- @@ -44,12 +50,14 @@ processors: S3ACLREQUIRED: "(-|Yes)" S3REGION: "[a-zA-Z][a-zA-Z0-9-]*" - grok: + tag: grok_aws_s3access_host_header field: aws.s3access.host_header ignore_missing: true ignore_failure: true patterns: - ^%{DATA}s3\.%{DATA:cloud.region}\.%{DATA}$ - script: + tag: script description: Drops null/empty values recursively lang: painless source: | @@ -67,17 +75,20 @@ processors: } drop(ctx); - grok: + tag: grok_aws_s3access_request_uri field: aws.s3access.request_uri ignore_failure: true patterns: - '%{NOTSPACE:http.request.method} %{NOTSPACE:_temp_.url} [hH][tT][tT][pP]/%{NOTSPACE:http.version}' - uri_parts: + tag: uri_parts_temp_url field: _temp_.url target_field: url keep_original: true ignore_failure: true if: ctx._temp_?.url != null - append: + tag: append_related_user field: related.user value: '{{aws.s3access.bucket_owner}}' allow_duplicates: false @@ -86,75 +97,92 @@ processors: # Parse the date included in s3 access logs # - date: + tag: date_temp_s3access_time field: _temp_.s3access_time target_field: '@timestamp' ignore_failure: true formats: - dd/MMM/yyyy:H:m:s Z - set: + tag: set_client_ip field: client.ip value: '{{aws.s3access.remote_ip}}' ignore_empty_value: true - append: + tag: append_related_ip field: related.ip value: '{{aws.s3access.remote_ip}}' allow_duplicates: false if: ctx?.aws?.s3access?.remote_ip != null - set: + tag: set_client_address field: client.address value: '{{aws.s3access.remote_ip}}' ignore_empty_value: true - geoip: + tag: geoip_aws_s3access_remote_ip field: aws.s3access.remote_ip target_field: client.geo if: ctx?.aws?.s3access?.remote_ip != null - set: + tag: set_geo field: geo copy_from: client.geo ignore_empty_value: true - set: + tag: set_client_user_id field: client.user.id value: '{{aws.s3access.requester}}' ignore_empty_value: true - set: + tag: set_event_id field: event.id value: '{{aws.s3access.request_id}}' ignore_empty_value: true - set: + tag: set_event_action field: event.action value: '{{aws.s3access.operation}}' ignore_empty_value: true - set: + tag: set_http_response_status_code field: http.response.status_code value: '{{aws.s3access.http_status}}' ignore_empty_value: true - convert: + tag: convert_http_response_status_code field: http.response.status_code type: long if: ctx?.http?.response?.status_code != null - set: + tag: set_event_outcome field: event.outcome value: failure if: ctx?.aws?.s3access?.error_code != null - set: + tag: set_event_code field: event.code value: '{{aws.s3access.error_code}}' ignore_empty_value: true - set: + tag: set_event_outcome_1 field: event.outcome value: success if: ctx?.aws?.s3access?.error_code == null - convert: + tag: convert_aws_s3access_bytes_sent field: aws.s3access.bytes_sent target_field: http.response.body.bytes type: long ignore_failure: true - convert: + tag: convert_aws_s3access_total_time field: aws.s3access.total_time target_field: event.duration type: long ignore_failure: true - script: + tag: script_1 lang: painless if: ctx.event?.duration != null params: @@ -162,17 +190,21 @@ processors: source: >- ctx.event.duration *= params.MS_TO_NS; - set: + tag: set_http_request_referrer field: http.request.referrer value: '{{aws.s3access.referrer}}' ignore_empty_value: true - user_agent: + tag: user_agent_aws_s3access_user_agent if: ctx?.aws?.s3access?.user_agent != null field: aws.s3access.user_agent - set: + tag: set_tls_cipher field: tls.cipher value: '{{aws.s3access.cipher_suite}}' ignore_empty_value: true - script: + tag: script_2 lang: painless if: ctx.aws?.s3access?.tls_version != null source: >- @@ -183,30 +215,37 @@ processors: ctx.tls.version = parts[1]; ctx.tls.version_protocol = parts[0] - set: + tag: set_aws_s3access_access_point_arn field: aws.s3access.access_point_arn value: '{{aws.s3access.access_point_arn}}' ignore_empty_value: true - set: + tag: set_aws_s3access_aclrequired field: aws.s3access.aclrequired value: '{{aws.s3access.aclrequired}}' ignore_empty_value: true - set: + tag: set_aws_s3access_source_region field: aws.s3access.source_region value: '{{aws.s3access.source_region}}' ignore_empty_value: true - set: + tag: set_cloud_provider field: cloud.provider value: aws - set: + tag: set_event_kind field: event.kind value: event # # Remove temporary fields # - remove: + tag: remove_temp field: _temp_ ignore_missing: true - script: + tag: script_3 lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | @@ -232,11 +271,13 @@ processors: handleMap(ctx); on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/securityhub_findings/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/securityhub_findings/elasticsearch/ingest_pipeline/default.yml index 9f64529195d..1c232794dde 100644 --- a/packages/aws/data_stream/securityhub_findings/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/securityhub_findings/elasticsearch/ingest_pipeline/default.yml @@ -13,9 +13,11 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - set: + tag: set_event_kind field: event.kind value: state - append: @@ -29,20 +31,24 @@ processors: tag: append_event_category allow_duplicates: false - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original field: event.original target_field: json ignore_failure: true - fingerprint: + tag: fingerprint fields: - json.UpdatedAt - json.Id @@ -58,33 +64,41 @@ processors: value: aws tag: set_cloud_provider - rename: + tag: rename_json_action_actiontype field: json.Action.ActionType target_field: aws.securityhub_findings.action.type ignore_missing: true - set: + tag: set_event_action field: event.action copy_from: aws.securityhub_findings.action.type ignore_failure: true - lowercase: + tag: lowercase_event_action field: event.action ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_affectedresources field: json.Action.AwsApiCallAction.AffectedResources target_field: aws.securityhub_findings.action.aws_api_call.affected_resources ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_api field: json.Action.AwsApiCallAction.Api target_field: aws.securityhub_findings.action.aws_api_call.api ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_callertype field: json.Action.AwsApiCallAction.CallerType target_field: aws.securityhub_findings.action.aws_api_call.caller.type ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_domaindetails_domain field: json.Action.AwsApiCallAction.DomainDetails.Domain target_field: aws.securityhub_findings.action.aws_api_call.domain_details.domain ignore_missing: true - date: + tag: date_json_action_awsapicallaction_firstseen field: json.Action.AwsApiCallAction.FirstSeen if: ctx.json?.Action?.AwsApiCallAction?.FirstSeen != null && ctx.json?.Action?.AwsApiCallAction?.FirstSeen != '' target_field: aws.securityhub_findings.action.aws_api_call.first_seen @@ -93,9 +107,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_action_awsapicallaction_lastseen field: json.Action.AwsApiCallAction.LastSeen if: ctx.json?.Action?.AwsApiCallAction?.LastSeen != null && ctx.json?.Action?.AwsApiCallAction?.LastSeen != '' target_field: aws.securityhub_findings.action.aws_api_call.last_seen @@ -104,21 +120,26 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_1 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_city_cityname field: json.Action.AwsApiCallAction.RemoteIpDetails.City.CityName target_field: aws.securityhub_findings.action.aws_api_call.remote_ip.city.name ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_country_countrycode field: json.Action.AwsApiCallAction.RemoteIpDetails.Country.CountryCode target_field: aws.securityhub_findings.action.aws_api_call.remote_ip.country.code ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_country_countryname field: json.Action.AwsApiCallAction.RemoteIpDetails.Country.CountryName target_field: aws.securityhub_findings.action.aws_api_call.remote_ip.country.name ignore_missing: true - convert: + tag: convert_json_action_awsapicallaction_remoteipdetails_geolocation_lat field: json.Action.AwsApiCallAction.RemoteIpDetails.GeoLocation.Lat target_field: aws.securityhub_findings.action.aws_api_call.remote_ip.geolocation.latitude if: ctx.json?.Action?.AwsApiCallAction?.RemoteIpDetails?.GeoLocation?.Lat != '' @@ -126,9 +147,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_awsapicallaction_remoteipdetails_geolocation_lon field: json.Action.AwsApiCallAction.RemoteIpDetails.GeoLocation.Lon target_field: aws.securityhub_findings.action.aws_api_call.remote_ip.geolocation.longitude if: ctx.json?.Action?.AwsApiCallAction?.RemoteIpDetails?.GeoLocation?.Lon != '' @@ -136,9 +159,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_awsapicallaction_remoteipdetails_ipaddressv4 field: json.Action.AwsApiCallAction.RemoteIpDetails.IpAddressV4 target_field: aws.securityhub_findings.action.aws_api_call.remote_ip.ip.address_v4 if: ctx.json?.Action?.AwsApiCallAction?.RemoteIpDetails?.IpAddressV4 != '' @@ -146,9 +171,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_awsapicallaction_remoteipdetails_organization_asn field: json.Action.AwsApiCallAction.RemoteIpDetails.Organization.Asn target_field: aws.securityhub_findings.action.aws_api_call.remote_ip.organization.asn if: ctx.json?.Action?.AwsApiCallAction?.RemoteIpDetails?.Organization?.Asn != '' @@ -156,25 +183,31 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_organization_asnorg field: json.Action.AwsApiCallAction.RemoteIpDetails.Organization.AsnOrg target_field: aws.securityhub_findings.action.aws_api_call.remote_ip.organization.asn_organization ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_organization_isp field: json.Action.AwsApiCallAction.RemoteIpDetails.Organization.Isp target_field: aws.securityhub_findings.action.aws_api_call.remote_ip.organization.internet_service_provider ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_organization_org field: json.Action.AwsApiCallAction.RemoteIpDetails.Organization.Org target_field: aws.securityhub_findings.action.aws_api_call.remote_ip.organization.internet_provider ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_servicename field: json.Action.AwsApiCallAction.ServiceName target_field: aws.securityhub_findings.action.aws_api_call.service.name ignore_missing: true - convert: + tag: convert_json_action_dnsrequestaction_blocked field: json.Action.DnsRequestAction.Blocked target_field: aws.securityhub_findings.action.dns_request.blocked if: ctx.json?.Action?.DnsRequestAction?.Blocked != '' @@ -182,17 +215,21 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_dnsrequestaction_domain field: json.Action.DnsRequestAction.Domain target_field: aws.securityhub_findings.action.dns_request.domain ignore_missing: true - rename: + tag: rename_json_action_dnsrequestaction_protocol field: json.Action.DnsRequestAction.Protocol target_field: aws.securityhub_findings.action.dns_request.protocol ignore_missing: true - convert: + tag: convert_json_action_networkconnectionaction_blocked field: json.Action.NetworkConnectionAction.Blocked target_field: aws.securityhub_findings.action.network_connection.blocked if: ctx.json?.Action?.NetworkConnectionAction?.Blocked != '' @@ -200,13 +237,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_networkconnectionaction_connectiondirection field: json.Action.NetworkConnectionAction.ConnectionDirection target_field: aws.securityhub_findings.action.network_connection.direction ignore_missing: true - convert: + tag: convert_json_action_networkconnectionaction_localportdetails_port field: json.Action.NetworkConnectionAction.LocalPortDetails.Port target_field: aws.securityhub_findings.action.network_connection.local.port.number if: ctx.json?.Action?.NetworkConnectionAction?.LocalPortDetails?.Port != '' @@ -214,29 +254,36 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_networkconnectionaction_localportdetails_portname field: json.Action.NetworkConnectionAction.LocalPortDetails.PortName target_field: aws.securityhub_findings.action.network_connection.local.port.name ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_protocol field: json.Action.NetworkConnectionAction.Protocol target_field: aws.securityhub_findings.action.network_connection.protocol ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_city_cityname field: json.Action.NetworkConnectionAction.RemoteIpDetails.City.CityName target_field: aws.securityhub_findings.action.network_connection.remote_ip.city.name ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_country_countrycode field: json.Action.NetworkConnectionAction.RemoteIpDetails.Country.CountryCode target_field: aws.securityhub_findings.action.network_connection.remote_ip.country.code ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_country_countryname field: json.Action.NetworkConnectionAction.RemoteIpDetails.Country.CountryName target_field: aws.securityhub_findings.action.network_connection.remote_ip.country.name ignore_missing: true - convert: + tag: convert_json_action_networkconnectionaction_remoteipdetails_geolocation_lat field: json.Action.NetworkConnectionAction.RemoteIpDetails.GeoLocation.Lat target_field: aws.securityhub_findings.action.network_connection.remote_ip.geolocation.latitude if: ctx.json?.Action?.NetworkConnectionAction?.RemoteIpDetails?.GeoLocation?.Lat != '' @@ -244,9 +291,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_networkconnectionaction_remoteipdetails_geolocation_lon field: json.Action.NetworkConnectionAction.RemoteIpDetails.GeoLocation.Lon target_field: aws.securityhub_findings.action.network_connection.remote_ip.geolocation.longitude if: ctx.json?.Action?.NetworkConnectionAction?.RemoteIpDetails?.GeoLocation?.Lon != '' @@ -254,9 +303,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_10 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_networkconnectionaction_remoteipdetails_ipaddressv4 field: json.Action.NetworkConnectionAction.RemoteIpDetails.IpAddressV4 target_field: aws.securityhub_findings.action.network_connection.remote_ip.ip.address_v4 if: ctx.json?.Action?.NetworkConnectionAction?.RemoteIpDetails?.IpAddressV4 != '' @@ -264,9 +315,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_11 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_networkconnectionaction_remoteipdetails_organization_asn field: json.Action.NetworkConnectionAction.RemoteIpDetails.Organization.Asn target_field: aws.securityhub_findings.action.network_connection.remote_ip.organization.asn if: ctx.json?.Action?.NetworkConnectionAction?.RemoteIpDetails?.Organization?.Asn != '' @@ -274,21 +327,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_12 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_organization_asnorg field: json.Action.NetworkConnectionAction.RemoteIpDetails.Organization.AsnOrg target_field: aws.securityhub_findings.action.network_connection.remote_ip.organization.asn_organization ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_organization_isp field: json.Action.NetworkConnectionAction.RemoteIpDetails.Organization.Isp target_field: aws.securityhub_findings.action.network_connection.remote_ip.organization.internet_service_provider ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_organization_org field: json.Action.NetworkConnectionAction.RemoteIpDetails.Organization.Org target_field: aws.securityhub_findings.action.network_connection.remote_ip.organization.internet_provider ignore_missing: true - convert: + tag: convert_json_action_networkconnectionaction_remoteportdetails_port field: json.Action.NetworkConnectionAction.RemotePortDetails.Port target_field: aws.securityhub_findings.action.network_connection.remote.port.number if: ctx.json?.Action?.NetworkConnectionAction?.RemotePortDetails?.Port != '' @@ -296,13 +354,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_13 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_networkconnectionaction_remoteportdetails_portname field: json.Action.NetworkConnectionAction.RemotePortDetails.PortName target_field: aws.securityhub_findings.action.network_connection.remote.port.name ignore_missing: true - convert: + tag: convert_json_action_portprobeaction_blocked field: json.Action.PortProbeAction.Blocked target_field: aws.securityhub_findings.action.port_probe.blocked if: ctx.json?.Action?.PortProbeAction?.Blocked != '' @@ -310,9 +371,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_14 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -322,11 +385,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_15 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_1 field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -336,11 +401,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_16 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_2 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -350,6 +417,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_3 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -359,6 +427,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_4 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -368,6 +437,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_5 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -377,6 +447,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_6 field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -386,11 +457,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_17 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_7 field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -400,11 +473,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_18 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_8 field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -414,11 +489,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_19 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_9 field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -428,11 +505,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_20 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_10 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -442,6 +521,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_11 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -451,6 +531,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_12 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -460,6 +541,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_13 field: json.Action.PortProbeAction.PortProbeDetails processor: remove: @@ -471,26 +553,32 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - rename: + tag: rename_json_action_portprobeaction_portprobedetails field: json.Action.PortProbeAction.PortProbeDetails target_field: aws.securityhub_findings.action.port_probe.details ignore_missing: true - rename: + tag: rename_json_awsaccountid field: json.AwsAccountId target_field: aws.securityhub_findings.aws_account_id ignore_missing: true - set: + tag: set_cloud_account_id field: cloud.account.id copy_from: aws.securityhub_findings.aws_account_id ignore_failure: true - rename: + tag: rename_json_companyname field: json.CompanyName target_field: aws.securityhub_findings.company.name ignore_missing: true - set: + tag: set_organization_name field: organization.name copy_from: aws.securityhub_findings.company.name ignore_failure: true - rename: + tag: rename_json_compliance_relatedrequirements field: json.Compliance.RelatedRequirements target_field: aws.securityhub_findings.compliance.related_requirements ignore_missing: true @@ -505,6 +593,7 @@ processors: tag: append_related_requirements_rule_ruleset allow_duplicates: false - rename: + tag: rename_json_compliance_status field: json.Compliance.Status target_field: aws.securityhub_findings.compliance.status ignore_missing: true @@ -544,6 +633,7 @@ processors: value: unknown if: ctx.event?.outcome == null - foreach: + tag: foreach_json_compliance_statusreasons field: json.Compliance.StatusReasons processor: rename: @@ -553,6 +643,7 @@ processors: ignore_failure: true if: ctx.json?.Compliance?.StatusReasons != null && ctx.json?.Compliance?.StatusReasons instanceof List - foreach: + tag: foreach_json_compliance_statusreasons_1 field: json.Compliance.StatusReasons processor: rename: @@ -562,10 +653,12 @@ processors: ignore_failure: true if: ctx.json?.Compliance?.StatusReasons != null && ctx.json?.Compliance?.StatusReasons instanceof List - rename: + tag: rename_json_compliance_statusreasons field: json.Compliance.StatusReasons target_field: aws.securityhub_findings.compliance.status_reasons ignore_missing: true - convert: + tag: convert_json_confidence field: json.Confidence target_field: aws.securityhub_findings.confidence if: ctx.json?.Confidence != '' @@ -573,9 +666,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_21 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_createdat field: json.CreatedAt if: ctx.json?.CreatedAt != null && ctx.json?.CreatedAt != '' target_field: aws.securityhub_findings.created_at @@ -584,6 +679,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_22 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: @@ -596,6 +692,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_23 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -604,6 +701,7 @@ processors: tag: set_timestamp ignore_empty_value: true - convert: + tag: convert_json_criticality field: json.Criticality target_field: aws.securityhub_findings.criticality if: ctx.json?.Criticality != '' @@ -611,9 +709,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_24 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_description field: json.Description target_field: aws.securityhub_findings.description ignore_missing: true @@ -623,6 +723,7 @@ processors: copy_from: aws.securityhub_findings.description ignore_empty_value: true - convert: + tag: convert_json_findingproviderfields_confidence field: json.FindingProviderFields.Confidence target_field: aws.securityhub_findings.provider_fields.confidence if: ctx.json?.FindingProviderFields?.Confidence != '' @@ -630,9 +731,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_25 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_findingproviderfields_criticality field: json.FindingProviderFields.Criticality target_field: aws.securityhub_findings.provider_fields.criticality if: ctx.json?.FindingProviderFields?.Criticality != '' @@ -640,9 +743,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_26 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_findingproviderfields_relatedfindings field: json.FindingProviderFields.RelatedFindings processor: rename: @@ -652,6 +757,7 @@ processors: ignore_failure: true if: ctx.json?.FindingProviderFields?.RelatedFindings != null && ctx.json?.FindingProviderFields?.RelatedFindings instanceof List - foreach: + tag: foreach_json_findingproviderfields_relatedfindings_1 field: json.FindingProviderFields.RelatedFindings processor: rename: @@ -661,18 +767,22 @@ processors: ignore_failure: true if: ctx.json?.FindingProviderFields?.RelatedFindings != null && ctx.json?.FindingProviderFields?.RelatedFindings instanceof List - rename: + tag: rename_json_findingproviderfields_relatedfindings field: json.FindingProviderFields.RelatedFindings target_field: aws.securityhub_findings.provider_fields.related_findings ignore_missing: true - rename: + tag: rename_json_findingproviderfields_severity_label field: json.FindingProviderFields.Severity.Label target_field: aws.securityhub_findings.provider_fields.severity.label ignore_missing: true - rename: + tag: rename_json_findingproviderfields_severity_original field: json.FindingProviderFields.Severity.Original target_field: aws.securityhub_findings.provider_fields.severity.original ignore_missing: true - convert: + tag: convert_json_findingproviderfields_severity_normalized field: json.FindingProviderFields.Severity.Normalized target_field: aws.securityhub_findings.provider_fields.severity.normalized if: ctx.json?.FindingProviderFields?.Severity?.Normalized != '' @@ -680,9 +790,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_27 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_findingproviderfields_severity_product field: json.FindingProviderFields.Severity.Product target_field: aws.securityhub_findings.provider_fields.severity.product if: ctx.json?.FindingProviderFields?.Severity?.Product != '' @@ -690,13 +802,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_28 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_findingproviderfields_types field: json.FindingProviderFields.Types target_field: aws.securityhub_findings.provider_fields.types ignore_missing: true - date: + tag: date_json_firstobservedat field: json.FirstObservedAt if: ctx.json?.FirstObservedAt != null && ctx.json?.FirstObservedAt != '' target_field: aws.securityhub_findings.first_observed_at @@ -705,9 +820,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_29 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_generatorid field: json.GeneratorId target_field: aws.securityhub_findings.generator.id ignore_missing: true @@ -717,6 +834,7 @@ processors: copy_from: aws.securityhub_findings.generator.id ignore_empty_value: true - rename: + tag: rename_json_compliance_securitycontrolid field: json.Compliance.SecurityControlId target_field: aws.securityhub_findings.compliance.security_control_id ignore_missing: true @@ -727,14 +845,17 @@ processors: if: ctx.rule?.id == null ignore_empty_value: true - rename: + tag: rename_json_id field: json.Id target_field: aws.securityhub_findings.id ignore_missing: true - set: + tag: set_event_id field: event.id copy_from: aws.securityhub_findings.id ignore_failure: true - date: + tag: date_json_lastobservedat field: json.LastObservedAt if: ctx.json?.LastObservedAt != null && ctx.json?.LastObservedAt != '' target_field: aws.securityhub_findings.last_observed_at @@ -743,6 +864,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_30 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: @@ -755,6 +877,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_31 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -763,6 +886,7 @@ processors: copy_from: aws.securityhub_findings.processed_at ignore_empty_value: true - foreach: + tag: foreach_json_malware field: json.Malware processor: rename: @@ -772,6 +896,7 @@ processors: ignore_failure: true if: ctx.json?.Malware != null && ctx.json?.Malware instanceof List - foreach: + tag: foreach_json_malware_1 field: json.Malware processor: rename: @@ -781,6 +906,7 @@ processors: ignore_failure: true if: ctx.json?.Malware != null && ctx.json?.Malware instanceof List - foreach: + tag: foreach_json_malware_2 field: json.Malware processor: rename: @@ -790,6 +916,7 @@ processors: ignore_failure: true if: ctx.json?.Malware != null && ctx.json?.Malware instanceof List - foreach: + tag: foreach_json_malware_3 field: json.Malware processor: rename: @@ -799,18 +926,22 @@ processors: ignore_failure: true if: ctx.json?.Malware != null && ctx.json?.Malware instanceof List - rename: + tag: rename_json_malware field: json.Malware target_field: aws.securityhub_findings.malware ignore_missing: true - rename: + tag: rename_json_network_destinationdomain field: json.Network.DestinationDomain target_field: aws.securityhub_findings.network.destination.domain ignore_missing: true - set: + tag: set_destination_domain field: destination.domain copy_from: aws.securityhub_findings.network.destination.domain ignore_failure: true - convert: + tag: convert_json_network_destinationipv4 field: json.Network.DestinationIpV4 target_field: aws.securityhub_findings.network.destination.ip.v4 if: ctx.json?.Network?.DestinationIpV4 != '' @@ -818,15 +949,18 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_32 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_destination_ip field: destination.ip value: '{{{aws.securityhub_findings.network.destination.ip.v4}}}' if: ctx.aws?.securityhub_findings?.network?.destination?.ip?.v4 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_network_destinationipv6 field: json.Network.DestinationIpV6 target_field: aws.securityhub_findings.network.destination.ip.v6 if: ctx.json?.Network?.DestinationIpV6 != '' @@ -834,15 +968,18 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_33 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_destination_ip_1 field: destination.ip value: '{{{aws.securityhub_findings.network.destination.ip.v6}}}' if: ctx.aws?.securityhub_findings?.network?.destination?.ip?.v6 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_network_destinationport field: json.Network.DestinationPort target_field: aws.securityhub_findings.network.destination.port if: ctx.json?.Network?.DestinationPort != '' @@ -850,25 +987,31 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_34 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_destination_port field: destination.port copy_from: aws.securityhub_findings.network.destination.port ignore_failure: true - rename: + tag: rename_json_network_direction field: json.Network.Direction target_field: aws.securityhub_findings.network.direction ignore_missing: true - set: + tag: set_network_direction field: network.direction value: inbound if: "ctx.aws?.securityhub_findings?.network?.direction == 'IN'" - set: + tag: set_network_direction_1 field: network.direction value: outbound if: "ctx.aws?.securityhub_findings?.network?.direction == 'OUT'" - convert: + tag: convert_json_network_openportrange_begin field: json.Network.OpenPortRange.Begin target_field: aws.securityhub_findings.network.open_port_range.begin if: ctx.json?.Network?.OpenPortRange?.Begin != '' @@ -876,9 +1019,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_35 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_network_openportrange_end field: json.Network.OpenPortRange.End target_field: aws.securityhub_findings.network.open_port_range.end if: ctx.json?.Network?.OpenPortRange?.End != '' @@ -886,28 +1031,35 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_36 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_network_protocol field: json.Network.Protocol target_field: aws.securityhub_findings.network.protocol ignore_missing: true - set: + tag: set_network_protocol field: network.protocol copy_from: aws.securityhub_findings.network.protocol ignore_failure: true - lowercase: + tag: lowercase_network_protocol field: network.protocol ignore_missing: true - rename: + tag: rename_json_network_sourcedomain field: json.Network.SourceDomain target_field: aws.securityhub_findings.network.source.domain ignore_missing: true - set: + tag: set_source_domain field: source.domain copy_from: aws.securityhub_findings.network.source.domain ignore_failure: true - convert: + tag: convert_json_network_sourceipv4 field: json.Network.SourceIpV4 target_field: aws.securityhub_findings.network.source.ip.v4 if: ctx.json?.Network?.SourceIpV4 != '' @@ -915,15 +1067,18 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_37 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_source_ip field: source.ip value: '{{{aws.securityhub_findings.network.source.ip.v4}}}' if: ctx.aws?.securityhub_findings?.network?.source?.ip?.v4 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_network_sourceipv6 field: json.Network.SourceIpV6 target_field: aws.securityhub_findings.network.source.ip.v6 if: ctx.json?.Network?.SourceIpV6 != '' @@ -931,31 +1086,38 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_38 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_source_ip_1 field: source.ip value: '{{{aws.securityhub_findings.network.source.ip.v6}}}' if: ctx.aws?.securityhub_findings?.network?.source?.ip?.v6 != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_network_sourcemac field: json.Network.SourceMac target_field: aws.securityhub_findings.network.source.mac ignore_missing: true - gsub: + tag: gsub_aws_securityhub_findings_network_source_mac field: aws.securityhub_findings.network.source.mac pattern: '[-:.]' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_aws_securityhub_findings_network_source_mac field: aws.securityhub_findings.network.source.mac ignore_missing: true - set: + tag: set_source_mac field: source.mac copy_from: aws.securityhub_findings.network.source.mac ignore_failure: true - convert: + tag: convert_json_network_sourceport field: json.Network.SourcePort target_field: aws.securityhub_findings.network.source.port if: ctx.json?.Network?.SourcePort != '' @@ -963,13 +1125,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_39 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_source_port field: source.port copy_from: aws.securityhub_findings.network.source.port ignore_failure: true - foreach: + tag: foreach_json_networkpath field: json.NetworkPath processor: rename: @@ -979,6 +1144,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_1 field: json.NetworkPath processor: rename: @@ -988,6 +1154,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_2 field: json.NetworkPath processor: rename: @@ -997,6 +1164,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_3 field: json.NetworkPath processor: foreach: @@ -1009,6 +1177,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_40 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1016,6 +1185,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_4 field: json.NetworkPath processor: foreach: @@ -1028,6 +1198,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_41 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1035,6 +1206,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_5 field: json.NetworkPath processor: foreach: @@ -1050,6 +1222,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_6 field: json.NetworkPath processor: rename: @@ -1059,6 +1232,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_7 field: json.NetworkPath processor: rename: @@ -1068,6 +1242,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_8 field: json.NetworkPath processor: rename: @@ -1077,6 +1252,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_9 field: json.NetworkPath processor: foreach: @@ -1089,6 +1265,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_42 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1096,6 +1273,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_10 field: json.NetworkPath processor: foreach: @@ -1108,6 +1286,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_43 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1115,6 +1294,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_11 field: json.NetworkPath processor: foreach: @@ -1130,6 +1310,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_12 field: json.NetworkPath processor: rename: @@ -1139,6 +1320,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_13 field: json.NetworkPath processor: rename: @@ -1148,6 +1330,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_14 field: json.NetworkPath processor: foreach: @@ -1160,6 +1343,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_44 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1167,6 +1351,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_15 field: json.NetworkPath processor: foreach: @@ -1179,6 +1364,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_45 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1186,6 +1372,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_16 field: json.NetworkPath processor: foreach: @@ -1201,6 +1388,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_17 field: json.NetworkPath processor: rename: @@ -1210,6 +1398,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_18 field: json.NetworkPath processor: rename: @@ -1219,6 +1408,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_19 field: json.NetworkPath processor: rename: @@ -1228,6 +1418,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_20 field: json.NetworkPath processor: foreach: @@ -1240,6 +1431,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_46 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1248,6 +1440,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_21 field: json.NetworkPath processor: foreach: @@ -1260,6 +1453,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_47 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1267,6 +1461,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_22 field: json.NetworkPath processor: foreach: @@ -1282,6 +1477,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_23 field: json.NetworkPath processor: rename: @@ -1291,14 +1487,17 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - rename: + tag: rename_json_networkpath field: json.NetworkPath target_field: aws.securityhub_findings.network_path ignore_missing: true - rename: + tag: rename_json_note_text field: json.Note.Text target_field: aws.securityhub_findings.note.text ignore_missing: true - date: + tag: date_json_note_updatedat field: json.Note.UpdatedAt if: ctx.json?.Note?.UpdatedAt != null && ctx.json?.Note?.UpdatedAt != '' target_field: aws.securityhub_findings.note.updated_at @@ -1307,13 +1506,16 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_48 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_note_updatedby field: json.Note.UpdatedBy target_field: aws.securityhub_findings.note.updated_by ignore_missing: true - convert: + tag: convert_json_patchsummary_failedcount field: json.PatchSummary.FailedCount target_field: aws.securityhub_findings.patch_summary.failed.count if: ctx.json?.PatchSummary?.FailedCount != '' @@ -1321,13 +1523,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_49 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_patchsummary_id field: json.PatchSummary.Id target_field: aws.securityhub_findings.patch_summary.id ignore_missing: true - convert: + tag: convert_json_patchsummary_installedcount field: json.PatchSummary.InstalledCount target_field: aws.securityhub_findings.patch_summary.installed.count if: ctx.json?.PatchSummary?.InstalledCount != '' @@ -1335,9 +1540,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_50 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_patchsummary_installedothercount field: json.PatchSummary.InstalledOtherCount target_field: aws.securityhub_findings.patch_summary.installed.other.count if: ctx.json?.PatchSummary?.InstalledOtherCount != '' @@ -1345,9 +1552,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_51 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_patchsummary_installedpendingreboot field: json.PatchSummary.InstalledPendingReboot target_field: aws.securityhub_findings.patch_summary.installed.pending_reboot if: ctx.json?.PatchSummary?.InstalledPendingReboot != '' @@ -1355,9 +1564,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_52 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_patchsummary_installedrejectedcount field: json.PatchSummary.InstalledRejectedCount target_field: aws.securityhub_findings.patch_summary.installed.rejected.count if: ctx.json?.PatchSummary?.InstalledRejectedCount != '' @@ -1365,9 +1576,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_53 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_patchsummary_missingcount field: json.PatchSummary.MissingCount target_field: aws.securityhub_findings.patch_summary.missing.count if: ctx.json?.PatchSummary?.MissingCount != '' @@ -1375,13 +1588,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_54 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_patchsummary_operation field: json.PatchSummary.Operation target_field: aws.securityhub_findings.patch_summary.operation.type ignore_missing: true - date: + tag: date_json_patchsummary_operationendtime field: json.PatchSummary.OperationEndTime if: ctx.json?.PatchSummary?.OperationEndTime != null && ctx.json?.PatchSummary?.OperationEndTime != '' target_field: aws.securityhub_findings.patch_summary.operation.end_time @@ -1390,9 +1606,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_55 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_patchsummary_operationstarttime field: json.PatchSummary.OperationStartTime if: ctx.json?.PatchSummary?.OperationStartTime != null && ctx.json?.PatchSummary?.OperationStartTime != '' target_field: aws.securityhub_findings.patch_summary.operation.start_time @@ -1401,13 +1619,16 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_56 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_patchsummary_rebootoption field: json.PatchSummary.RebootOption target_field: aws.securityhub_findings.patch_summary.reboot_option ignore_missing: true - date: + tag: date_json_process_launchedat field: json.Process.LaunchedAt if: ctx.json?.Process?.LaunchedAt != null && ctx.json?.Process?.LaunchedAt != '' target_field: aws.securityhub_findings.process.launched_at @@ -1416,21 +1637,26 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_57 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_process_start field: process.start copy_from: aws.securityhub_findings.process.launched_at ignore_failure: true - rename: + tag: rename_json_process_name field: json.Process.Name target_field: aws.securityhub_findings.process.name ignore_missing: true - set: + tag: set_process_name field: process.name copy_from: aws.securityhub_findings.process.name ignore_failure: true - convert: + tag: convert_json_process_parentpid field: json.Process.ParentPid target_field: aws.securityhub_findings.process.parent.pid if: ctx.json?.Process?.ParentPid != '' @@ -1438,21 +1664,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_58 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_process_parent_pid field: process.parent.pid copy_from: aws.securityhub_findings.process.parent.pid ignore_failure: true - rename: + tag: rename_json_process_path field: json.Process.Path target_field: aws.securityhub_findings.process.path ignore_missing: true - set: + tag: set_process_executable field: process.executable copy_from: aws.securityhub_findings.process.path ignore_failure: true - convert: + tag: convert_json_process_pid field: json.Process.Pid target_field: aws.securityhub_findings.process.pid if: ctx.json?.Process?.Pid != '' @@ -1460,13 +1691,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_59 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_process_pid field: process.pid copy_from: aws.securityhub_findings.process.pid ignore_failure: true - date: + tag: date_json_process_terminatedat field: json.Process.TerminatedAt if: ctx.json?.Process?.TerminatedAt != null && ctx.json?.Process?.TerminatedAt != '' target_field: aws.securityhub_findings.process.terminated_at @@ -1475,29 +1709,36 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_60 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_process_end field: process.end copy_from: aws.securityhub_findings.process.terminated_at ignore_failure: true - rename: + tag: rename_json_productarn field: json.ProductArn target_field: aws.securityhub_findings.product.arn ignore_missing: true - rename: + tag: rename_json_productfields field: json.ProductFields target_field: aws.securityhub_findings.product.fields ignore_missing: true - rename: + tag: rename_json_productname field: json.ProductName target_field: aws.securityhub_findings.product.name ignore_missing: true - rename: + tag: rename_json_recordstate field: json.RecordState target_field: aws.securityhub_findings.record_state ignore_missing: true - rename: + tag: rename_json_region field: json.Region target_field: aws.securityhub_findings.region ignore_missing: true @@ -1507,6 +1748,7 @@ processors: copy_from: aws.securityhub_findings.region ignore_empty_value: true - foreach: + tag: foreach_json_relatedfindings field: json.RelatedFindings processor: rename: @@ -1516,6 +1758,7 @@ processors: ignore_failure: true if: ctx.json?.RelatedFindings != null && ctx.json?.RelatedFindings instanceof List - foreach: + tag: foreach_json_relatedfindings_1 field: json.RelatedFindings processor: rename: @@ -1525,14 +1768,17 @@ processors: ignore_failure: true if: ctx.json?.RelatedFindings != null && ctx.json?.RelatedFindings instanceof List - rename: + tag: rename_json_relatedfindings field: json.RelatedFindings target_field: aws.securityhub_findings.related_findings ignore_missing: true - rename: + tag: rename_json_remediation_recommendation_text field: json.Remediation.Recommendation.Text target_field: aws.securityhub_findings.remediation.recommendation.text ignore_missing: true - rename: + tag: rename_json_remediation_recommendation_url field: json.Remediation.Recommendation.Url target_field: aws.securityhub_findings.remediation.recommendation.url ignore_missing: true @@ -1548,6 +1794,7 @@ processors: if: ctx.aws?.securityhub_findings?.remediation?.recommendation?.url != null && ctx.aws.securityhub_findings.remediation.recommendation.text != null ignore_empty_value: true - rename: + tag: rename_json_resources field: json.Resources target_field: aws.securityhub_findings.resources ignore_missing: true @@ -1917,6 +2164,7 @@ processors: } } - convert: + tag: convert_json_sample field: json.Sample target_field: aws.securityhub_findings.sample if: ctx.json?.Sample != '' @@ -1924,17 +2172,21 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_61 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_schemaversion field: json.SchemaVersion target_field: aws.securityhub_findings.schema.version ignore_missing: true - rename: + tag: rename_json_severity_label field: json.Severity.Label target_field: aws.securityhub_findings.severity.label ignore_missing: true - convert: + tag: convert_json_severity_normalized field: json.Severity.Normalized target_field: aws.securityhub_findings.severity.normalized if: ctx.json?.Severity?.Normalized != '' @@ -1942,6 +2194,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_62 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: @@ -1952,13 +2205,16 @@ processors: type: long on_failure: - append: + tag: append_error_message_63 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_severity_original field: json.Severity.Original target_field: aws.securityhub_findings.severity.original ignore_missing: true - convert: + tag: convert_json_severity_product field: json.Severity.Product target_field: aws.securityhub_findings.severity.product if: ctx.json?.Severity?.Product != '' @@ -1966,24 +2222,30 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_64 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_sourceurl field: json.SourceUrl target_field: aws.securityhub_findings.source_url ignore_missing: true - uri_parts: + tag: uri_parts_aws_securityhub_findings_source_url field: aws.securityhub_findings.source_url if: ctx.aws?.securityhub_findings?.source_url != '' && ctx.aws?.securityhub_findings?.source_url != null on_failure: - append: + tag: append_error_message_65 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_url_full field: url.full value: '{{{url.original}}}' ignore_failure: true - foreach: + tag: foreach_json_threatintelindicators field: json.ThreatIntelIndicators processor: rename: @@ -1993,6 +2255,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_1 field: json.ThreatIntelIndicators processor: date: @@ -2005,6 +2268,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_2 field: json.ThreatIntelIndicators processor: remove: @@ -2014,6 +2278,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_3 field: json.ThreatIntelIndicators processor: rename: @@ -2023,6 +2288,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_4 field: json.ThreatIntelIndicators processor: rename: @@ -2032,6 +2298,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_5 field: json.ThreatIntelIndicators processor: rename: @@ -2041,6 +2308,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_6 field: json.ThreatIntelIndicators processor: rename: @@ -2050,6 +2318,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_7 field: json.ThreatIntelIndicators processor: set: @@ -2059,6 +2328,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - script: + tag: script description: Map box field ThreatIntelIndicator to ECS field threat.indicator.type lang: painless params: @@ -2102,10 +2372,12 @@ processors: } if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - rename: + tag: rename_json_threatintelindicators field: json.ThreatIntelIndicators target_field: aws.securityhub_findings.threat_intel_indicators ignore_missing: true - rename: + tag: rename_json_title field: json.Title target_field: aws.securityhub_findings.title ignore_missing: true @@ -2115,18 +2387,22 @@ processors: copy_from: aws.securityhub_findings.title ignore_empty_value: true - rename: + tag: rename_json_types field: json.Types target_field: aws.securityhub_findings.types ignore_missing: true - rename: + tag: rename_json_userdefinedfields field: json.UserDefinedFields target_field: aws.securityhub_findings.user_defined_fields ignore_missing: true - rename: + tag: rename_json_verificationstate field: json.VerificationState target_field: aws.securityhub_findings.verification_state ignore_missing: true - foreach: + tag: foreach_json_vulnerabilities field: json.Vulnerabilities processor: foreach: @@ -2146,6 +2422,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_1 field: json.Vulnerabilities processor: foreach: @@ -2165,6 +2442,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_2 field: json.Vulnerabilities processor: foreach: @@ -2179,6 +2457,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_3 field: json.Vulnerabilities processor: foreach: @@ -2191,6 +2470,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_66 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -2198,6 +2478,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_4 field: json.Vulnerabilities processor: foreach: @@ -2212,6 +2493,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_5 field: json.Vulnerabilities processor: foreach: @@ -2226,6 +2508,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_6 field: json.Vulnerabilities processor: foreach: @@ -2240,6 +2523,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_7 field: json.Vulnerabilities processor: foreach: @@ -2254,6 +2538,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_8 field: json.Vulnerabilities processor: foreach: @@ -2268,6 +2553,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_9 field: json.Vulnerabilities processor: foreach: @@ -2282,6 +2568,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_10 field: json.Vulnerabilities processor: rename: @@ -2291,6 +2578,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_11 field: json.Vulnerabilities processor: rename: @@ -2300,6 +2588,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_12 field: json.Vulnerabilities processor: set: @@ -2309,6 +2598,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_13 field: json.Vulnerabilities processor: rename: @@ -2318,6 +2608,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_14 field: json.Vulnerabilities processor: set: @@ -2327,6 +2618,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_15 field: json.Vulnerabilities processor: rename: @@ -2336,6 +2628,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_16 field: json.Vulnerabilities processor: rename: @@ -2345,6 +2638,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_17 field: json.Vulnerabilities processor: set: @@ -2354,6 +2648,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_18 field: json.Vulnerabilities processor: rename: @@ -2363,6 +2658,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_19 field: json.Vulnerabilities processor: date: @@ -2375,6 +2671,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_20 field: json.Vulnerabilities processor: rename: @@ -2384,6 +2681,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_21 field: json.Vulnerabilities processor: date: @@ -2396,6 +2694,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_22 field: json.Vulnerabilities processor: remove: @@ -2406,6 +2705,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_23 field: json.Vulnerabilities processor: foreach: @@ -2420,6 +2720,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_24 field: json.Vulnerabilities processor: foreach: @@ -2434,6 +2735,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_25 field: json.Vulnerabilities processor: foreach: @@ -2448,6 +2750,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_26 field: json.Vulnerabilities processor: foreach: @@ -2462,6 +2765,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_27 field: json.Vulnerabilities processor: foreach: @@ -2476,6 +2780,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_28 field: json.Vulnerabilities processor: foreach: @@ -2490,6 +2795,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_29 field: json.Vulnerabilities processor: foreach: @@ -2504,6 +2810,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_30 field: json.Vulnerabilities processor: foreach: @@ -2518,6 +2825,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_31 field: json.Vulnerabilities processor: rename: @@ -2527,34 +2835,41 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - rename: + tag: rename_json_vulnerabilities field: json.Vulnerabilities target_field: aws.securityhub_findings.vulnerabilities ignore_missing: true - rename: + tag: rename_json_workflow_status field: json.Workflow.Status target_field: aws.securityhub_findings.workflow.status ignore_missing: true - rename: + tag: rename_json_workflowstate field: json.WorkflowState target_field: aws.securityhub_findings.workflow.state ignore_missing: true - remove: + tag: remove field: - json ignore_missing: true - append: + tag: append_related_ip field: related.ip value: '{{{aws.securityhub_findings.action.aws_api_call.remote_ip.ip.address_v4}}}' if: ctx.aws?.securityhub_findings?.action?.aws_api_call?.remote_ip?.ip?.address_v4 != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_1 field: related.ip value: '{{{aws.securityhub_findings.action.network_connection.remote_ip.ip.address_v4}}}' if: ctx.aws?.securityhub_findings?.action?.network_connection?.remote_ip?.ip?.address_v4 != null allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_aws_securityhub_findings_action_port_probe_details field: aws.securityhub_findings.action.port_probe.details processor: append: @@ -2565,6 +2880,7 @@ processors: ignore_failure: true if: ctx.aws?.securityhub_findings?.action?.port_probe?.details != null && ctx.aws?.securityhub_findings?.action?.port_probe?.details instanceof List - foreach: + tag: foreach_aws_securityhub_findings_action_port_probe_details_1 field: aws.securityhub_findings.action.port_probe.details processor: append: @@ -2575,30 +2891,35 @@ processors: ignore_failure: true if: ctx.aws?.securityhub_findings?.action?.port_probe?.details != null && ctx.aws?.securityhub_findings?.action?.port_probe?.details instanceof List - append: + tag: append_related_ip_2 field: related.ip value: '{{{aws.securityhub_findings.network.destination.ip.v4}}}' if: ctx.aws?.securityhub_findings?.network?.destination?.ip?.v4 != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_3 field: related.ip value: '{{{aws.securityhub_findings.network.destination.ip.v6}}}' if: ctx.aws?.securityhub_findings?.network?.destination?.ip?.v6 != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_4 field: related.ip value: '{{{aws.securityhub_findings.network.source.ip.v4}}}' if: ctx.aws?.securityhub_findings?.network?.source?.ip?.v4 != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_5 field: related.ip value: '{{{aws.securityhub_findings.network.source.ip.v6}}}' if: ctx.aws?.securityhub_findings?.network?.source?.ip?.v6 != null allow_duplicates: false ignore_failure: true - remove: + tag: remove_1 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - aws.securityhub_findings.created_at @@ -2624,6 +2945,7 @@ processors: ignore_failure: true ignore_missing: true - foreach: + tag: foreach_aws_securityhub_findings_threat_intel_indicators field: aws.securityhub_findings.threat_intel_indicators processor: remove: @@ -2635,6 +2957,7 @@ processors: ignore_missing: true ignore_missing: true - foreach: + tag: foreach_aws_securityhub_findings_vulnerabilities field: aws.securityhub_findings.vulnerabilities processor: remove: @@ -2649,6 +2972,7 @@ processors: ignore_missing: true ignore_missing: true - script: + tag: script_1 description: Drops null/empty values recursively. lang: painless source: | @@ -2667,11 +2991,13 @@ processors: dropEmptyFields(ctx); on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message_67 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/securityhub_findings_full_posture/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/securityhub_findings_full_posture/elasticsearch/ingest_pipeline/default.yml index 9c9d22e9964..a16bacf39d8 100644 --- a/packages/aws/data_stream/securityhub_findings_full_posture/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/securityhub_findings_full_posture/elasticsearch/ingest_pipeline/default.yml @@ -13,9 +13,11 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - set: + tag: set_event_kind field: event.kind value: state - append: @@ -29,20 +31,24 @@ processors: tag: append_event_category allow_duplicates: false - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original field: event.original target_field: json ignore_failure: true - fingerprint: + tag: fingerprint fields: - json.UpdatedAt - json.Id @@ -58,33 +64,41 @@ processors: value: aws tag: set_cloud_provider - rename: + tag: rename_json_action_actiontype field: json.Action.ActionType target_field: aws.securityhub_findings_full_posture.action.type ignore_missing: true - set: + tag: set_event_action field: event.action copy_from: aws.securityhub_findings_full_posture.action.type ignore_failure: true - lowercase: + tag: lowercase_event_action field: event.action ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_affectedresources field: json.Action.AwsApiCallAction.AffectedResources target_field: aws.securityhub_findings_full_posture.action.aws_api_call.affected_resources ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_api field: json.Action.AwsApiCallAction.Api target_field: aws.securityhub_findings_full_posture.action.aws_api_call.api ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_callertype field: json.Action.AwsApiCallAction.CallerType target_field: aws.securityhub_findings_full_posture.action.aws_api_call.caller.type ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_domaindetails_domain field: json.Action.AwsApiCallAction.DomainDetails.Domain target_field: aws.securityhub_findings_full_posture.action.aws_api_call.domain_details.domain ignore_missing: true - date: + tag: date_json_action_awsapicallaction_firstseen field: json.Action.AwsApiCallAction.FirstSeen if: ctx.json?.Action?.AwsApiCallAction?.FirstSeen != null && ctx.json?.Action?.AwsApiCallAction?.FirstSeen != '' target_field: aws.securityhub_findings_full_posture.action.aws_api_call.first_seen @@ -93,9 +107,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_action_awsapicallaction_lastseen field: json.Action.AwsApiCallAction.LastSeen if: ctx.json?.Action?.AwsApiCallAction?.LastSeen != null && ctx.json?.Action?.AwsApiCallAction?.LastSeen != '' target_field: aws.securityhub_findings_full_posture.action.aws_api_call.last_seen @@ -104,21 +120,26 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_1 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_city_cityname field: json.Action.AwsApiCallAction.RemoteIpDetails.City.CityName target_field: aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.city.name ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_country_countrycode field: json.Action.AwsApiCallAction.RemoteIpDetails.Country.CountryCode target_field: aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.country.code ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_country_countryname field: json.Action.AwsApiCallAction.RemoteIpDetails.Country.CountryName target_field: aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.country.name ignore_missing: true - convert: + tag: convert_json_action_awsapicallaction_remoteipdetails_geolocation_lat field: json.Action.AwsApiCallAction.RemoteIpDetails.GeoLocation.Lat target_field: aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.geolocation.latitude if: ctx.json?.Action?.AwsApiCallAction?.RemoteIpDetails?.GeoLocation?.Lat != '' @@ -126,9 +147,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_awsapicallaction_remoteipdetails_geolocation_lon field: json.Action.AwsApiCallAction.RemoteIpDetails.GeoLocation.Lon target_field: aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.geolocation.longitude if: ctx.json?.Action?.AwsApiCallAction?.RemoteIpDetails?.GeoLocation?.Lon != '' @@ -136,9 +159,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_3 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_awsapicallaction_remoteipdetails_ipaddressv4 field: json.Action.AwsApiCallAction.RemoteIpDetails.IpAddressV4 target_field: aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.ip.address_v4 if: ctx.json?.Action?.AwsApiCallAction?.RemoteIpDetails?.IpAddressV4 != '' @@ -146,9 +171,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_awsapicallaction_remoteipdetails_organization_asn field: json.Action.AwsApiCallAction.RemoteIpDetails.Organization.Asn target_field: aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.organization.asn if: ctx.json?.Action?.AwsApiCallAction?.RemoteIpDetails?.Organization?.Asn != '' @@ -156,25 +183,31 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_organization_asnorg field: json.Action.AwsApiCallAction.RemoteIpDetails.Organization.AsnOrg target_field: aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.organization.asn_organization ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_organization_isp field: json.Action.AwsApiCallAction.RemoteIpDetails.Organization.Isp target_field: aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.organization.internet_service_provider ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_remoteipdetails_organization_org field: json.Action.AwsApiCallAction.RemoteIpDetails.Organization.Org target_field: aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.organization.internet_provider ignore_missing: true - rename: + tag: rename_json_action_awsapicallaction_servicename field: json.Action.AwsApiCallAction.ServiceName target_field: aws.securityhub_findings_full_posture.action.aws_api_call.service.name ignore_missing: true - convert: + tag: convert_json_action_dnsrequestaction_blocked field: json.Action.DnsRequestAction.Blocked target_field: aws.securityhub_findings_full_posture.action.dns_request.blocked if: ctx.json?.Action?.DnsRequestAction?.Blocked != '' @@ -182,17 +215,21 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_dnsrequestaction_domain field: json.Action.DnsRequestAction.Domain target_field: aws.securityhub_findings_full_posture.action.dns_request.domain ignore_missing: true - rename: + tag: rename_json_action_dnsrequestaction_protocol field: json.Action.DnsRequestAction.Protocol target_field: aws.securityhub_findings_full_posture.action.dns_request.protocol ignore_missing: true - convert: + tag: convert_json_action_networkconnectionaction_blocked field: json.Action.NetworkConnectionAction.Blocked target_field: aws.securityhub_findings_full_posture.action.network_connection.blocked if: ctx.json?.Action?.NetworkConnectionAction?.Blocked != '' @@ -200,13 +237,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_7 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_networkconnectionaction_connectiondirection field: json.Action.NetworkConnectionAction.ConnectionDirection target_field: aws.securityhub_findings_full_posture.action.network_connection.direction ignore_missing: true - convert: + tag: convert_json_action_networkconnectionaction_localportdetails_port field: json.Action.NetworkConnectionAction.LocalPortDetails.Port target_field: aws.securityhub_findings_full_posture.action.network_connection.local.port.number if: ctx.json?.Action?.NetworkConnectionAction?.LocalPortDetails?.Port != '' @@ -214,29 +254,36 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_8 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_networkconnectionaction_localportdetails_portname field: json.Action.NetworkConnectionAction.LocalPortDetails.PortName target_field: aws.securityhub_findings_full_posture.action.network_connection.local.port.name ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_protocol field: json.Action.NetworkConnectionAction.Protocol target_field: aws.securityhub_findings_full_posture.action.network_connection.protocol ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_city_cityname field: json.Action.NetworkConnectionAction.RemoteIpDetails.City.CityName target_field: aws.securityhub_findings_full_posture.action.network_connection.remote_ip.city.name ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_country_countrycode field: json.Action.NetworkConnectionAction.RemoteIpDetails.Country.CountryCode target_field: aws.securityhub_findings_full_posture.action.network_connection.remote_ip.country.code ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_country_countryname field: json.Action.NetworkConnectionAction.RemoteIpDetails.Country.CountryName target_field: aws.securityhub_findings_full_posture.action.network_connection.remote_ip.country.name ignore_missing: true - convert: + tag: convert_json_action_networkconnectionaction_remoteipdetails_geolocation_lat field: json.Action.NetworkConnectionAction.RemoteIpDetails.GeoLocation.Lat target_field: aws.securityhub_findings_full_posture.action.network_connection.remote_ip.geolocation.latitude if: ctx.json?.Action?.NetworkConnectionAction?.RemoteIpDetails?.GeoLocation?.Lat != '' @@ -244,9 +291,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_9 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_networkconnectionaction_remoteipdetails_geolocation_lon field: json.Action.NetworkConnectionAction.RemoteIpDetails.GeoLocation.Lon target_field: aws.securityhub_findings_full_posture.action.network_connection.remote_ip.geolocation.longitude if: ctx.json?.Action?.NetworkConnectionAction?.RemoteIpDetails?.GeoLocation?.Lon != '' @@ -254,9 +303,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_10 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_networkconnectionaction_remoteipdetails_ipaddressv4 field: json.Action.NetworkConnectionAction.RemoteIpDetails.IpAddressV4 target_field: aws.securityhub_findings_full_posture.action.network_connection.remote_ip.ip.address_v4 if: ctx.json?.Action?.NetworkConnectionAction?.RemoteIpDetails?.IpAddressV4 != '' @@ -264,9 +315,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_11 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_action_networkconnectionaction_remoteipdetails_organization_asn field: json.Action.NetworkConnectionAction.RemoteIpDetails.Organization.Asn target_field: aws.securityhub_findings_full_posture.action.network_connection.remote_ip.organization.asn if: ctx.json?.Action?.NetworkConnectionAction?.RemoteIpDetails?.Organization?.Asn != '' @@ -274,21 +327,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_12 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_organization_asnorg field: json.Action.NetworkConnectionAction.RemoteIpDetails.Organization.AsnOrg target_field: aws.securityhub_findings_full_posture.action.network_connection.remote_ip.organization.asn_organization ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_organization_isp field: json.Action.NetworkConnectionAction.RemoteIpDetails.Organization.Isp target_field: aws.securityhub_findings_full_posture.action.network_connection.remote_ip.organization.internet_service_provider ignore_missing: true - rename: + tag: rename_json_action_networkconnectionaction_remoteipdetails_organization_org field: json.Action.NetworkConnectionAction.RemoteIpDetails.Organization.Org target_field: aws.securityhub_findings_full_posture.action.network_connection.remote_ip.organization.internet_provider ignore_missing: true - convert: + tag: convert_json_action_networkconnectionaction_remoteportdetails_port field: json.Action.NetworkConnectionAction.RemotePortDetails.Port target_field: aws.securityhub_findings_full_posture.action.network_connection.remote.port.number if: ctx.json?.Action?.NetworkConnectionAction?.RemotePortDetails?.Port != '' @@ -296,13 +354,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_13 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action_networkconnectionaction_remoteportdetails_portname field: json.Action.NetworkConnectionAction.RemotePortDetails.PortName target_field: aws.securityhub_findings_full_posture.action.network_connection.remote.port.name ignore_missing: true - convert: + tag: convert_json_action_portprobeaction_blocked field: json.Action.PortProbeAction.Blocked target_field: aws.securityhub_findings_full_posture.action.port_probe.blocked if: ctx.json?.Action?.PortProbeAction?.Blocked != '' @@ -310,9 +371,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_14 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -322,11 +385,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_15 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_1 field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -336,11 +401,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_16 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_2 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -350,6 +417,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_3 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -359,6 +427,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_4 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -368,6 +437,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_5 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -377,6 +447,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_6 field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -386,11 +457,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_17 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_7 field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -400,11 +473,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_18 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_8 field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -414,11 +489,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_19 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_9 field: json.Action.PortProbeAction.PortProbeDetails processor: convert: @@ -428,11 +505,13 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_20 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_10 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -442,6 +521,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_11 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -451,6 +531,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_12 field: json.Action.PortProbeAction.PortProbeDetails processor: rename: @@ -460,6 +541,7 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - foreach: + tag: foreach_json_action_portprobeaction_portprobedetails_13 field: json.Action.PortProbeAction.PortProbeDetails processor: remove: @@ -471,26 +553,32 @@ processors: ignore_failure: true if: ctx.json?.Action?.PortProbeAction?.PortProbeDetails != null && ctx.json?.Action?.PortProbeAction?.PortProbeDetails instanceof List - rename: + tag: rename_json_action_portprobeaction_portprobedetails field: json.Action.PortProbeAction.PortProbeDetails target_field: aws.securityhub_findings_full_posture.action.port_probe.details ignore_missing: true - rename: + tag: rename_json_awsaccountid field: json.AwsAccountId target_field: aws.securityhub_findings_full_posture.aws_account_id ignore_missing: true - set: + tag: set_cloud_account_id field: cloud.account.id copy_from: aws.securityhub_findings_full_posture.aws_account_id ignore_failure: true - rename: + tag: rename_json_companyname field: json.CompanyName target_field: aws.securityhub_findings_full_posture.company.name ignore_missing: true - set: + tag: set_organization_name field: organization.name copy_from: aws.securityhub_findings_full_posture.company.name ignore_failure: true - rename: + tag: rename_json_compliance_relatedrequirements field: json.Compliance.RelatedRequirements target_field: aws.securityhub_findings_full_posture.compliance.related_requirements ignore_missing: true @@ -505,6 +593,7 @@ processors: tag: append_related_requirements_rule_ruleset allow_duplicates: false - rename: + tag: rename_json_compliance_status field: json.Compliance.Status target_field: aws.securityhub_findings_full_posture.compliance.status ignore_missing: true @@ -544,6 +633,7 @@ processors: value: unknown if: ctx.event?.outcome == null - foreach: + tag: foreach_json_compliance_statusreasons field: json.Compliance.StatusReasons processor: rename: @@ -553,6 +643,7 @@ processors: ignore_failure: true if: ctx.json?.Compliance?.StatusReasons != null && ctx.json?.Compliance?.StatusReasons instanceof List - foreach: + tag: foreach_json_compliance_statusreasons_1 field: json.Compliance.StatusReasons processor: rename: @@ -562,10 +653,12 @@ processors: ignore_failure: true if: ctx.json?.Compliance?.StatusReasons != null && ctx.json?.Compliance?.StatusReasons instanceof List - rename: + tag: rename_json_compliance_statusreasons field: json.Compliance.StatusReasons target_field: aws.securityhub_findings_full_posture.compliance.status_reasons ignore_missing: true - convert: + tag: convert_json_confidence field: json.Confidence target_field: aws.securityhub_findings_full_posture.confidence if: ctx.json?.Confidence != '' @@ -573,9 +666,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_21 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_createdat field: json.CreatedAt if: ctx.json?.CreatedAt != null && ctx.json?.CreatedAt != '' target_field: aws.securityhub_findings_full_posture.created_at @@ -584,6 +679,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_22 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: @@ -596,6 +692,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_23 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -603,6 +700,7 @@ processors: value: "{{{_ingest.timestamp}}}" tag: set_timestamp - convert: + tag: convert_json_criticality field: json.Criticality target_field: aws.securityhub_findings_full_posture.criticality if: ctx.json?.Criticality != '' @@ -610,9 +708,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_24 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_description field: json.Description target_field: aws.securityhub_findings_full_posture.description ignore_missing: true @@ -622,6 +722,7 @@ processors: copy_from: aws.securityhub_findings_full_posture.description ignore_empty_value: true - convert: + tag: convert_json_findingproviderfields_confidence field: json.FindingProviderFields.Confidence target_field: aws.securityhub_findings_full_posture.provider_fields.confidence if: ctx.json?.FindingProviderFields?.Confidence != '' @@ -629,9 +730,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_25 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_findingproviderfields_criticality field: json.FindingProviderFields.Criticality target_field: aws.securityhub_findings_full_posture.provider_fields.criticality if: ctx.json?.FindingProviderFields?.Criticality != '' @@ -639,9 +742,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_26 field: error.message value: '{{{_ingest.on_failure_message}}}' - foreach: + tag: foreach_json_findingproviderfields_relatedfindings field: json.FindingProviderFields.RelatedFindings processor: rename: @@ -651,6 +756,7 @@ processors: ignore_failure: true if: ctx.json?.FindingProviderFields?.RelatedFindings != null && ctx.json?.FindingProviderFields?.RelatedFindings instanceof List - foreach: + tag: foreach_json_findingproviderfields_relatedfindings_1 field: json.FindingProviderFields.RelatedFindings processor: rename: @@ -660,18 +766,22 @@ processors: ignore_failure: true if: ctx.json?.FindingProviderFields?.RelatedFindings != null && ctx.json?.FindingProviderFields?.RelatedFindings instanceof List - rename: + tag: rename_json_findingproviderfields_relatedfindings field: json.FindingProviderFields.RelatedFindings target_field: aws.securityhub_findings_full_posture.provider_fields.related_findings ignore_missing: true - rename: + tag: rename_json_findingproviderfields_severity_label field: json.FindingProviderFields.Severity.Label target_field: aws.securityhub_findings_full_posture.provider_fields.severity.label ignore_missing: true - rename: + tag: rename_json_findingproviderfields_severity_original field: json.FindingProviderFields.Severity.Original target_field: aws.securityhub_findings_full_posture.provider_fields.severity.original ignore_missing: true - convert: + tag: convert_json_findingproviderfields_severity_normalized field: json.FindingProviderFields.Severity.Normalized target_field: aws.securityhub_findings_full_posture.provider_fields.severity.normalized if: ctx.json?.FindingProviderFields?.Severity?.Normalized != '' @@ -679,9 +789,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_27 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_findingproviderfields_severity_product field: json.FindingProviderFields.Severity.Product target_field: aws.securityhub_findings_full_posture.provider_fields.severity.product if: ctx.json?.FindingProviderFields?.Severity?.Product != '' @@ -689,13 +801,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_28 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_findingproviderfields_types field: json.FindingProviderFields.Types target_field: aws.securityhub_findings_full_posture.provider_fields.types ignore_missing: true - date: + tag: date_json_firstobservedat field: json.FirstObservedAt if: ctx.json?.FirstObservedAt != null && ctx.json?.FirstObservedAt != '' target_field: aws.securityhub_findings_full_posture.first_observed_at @@ -704,9 +819,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_29 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_generatorid field: json.GeneratorId target_field: aws.securityhub_findings_full_posture.generator.id ignore_missing: true @@ -716,6 +833,7 @@ processors: copy_from: aws.securityhub_findings_full_posture.generator.id ignore_empty_value: true - rename: + tag: rename_json_compliance_securitycontrolid field: json.Compliance.SecurityControlId target_field: aws.securityhub_findings_full_posture.compliance.security_control_id ignore_missing: true @@ -726,14 +844,17 @@ processors: if: ctx.rule?.id == null ignore_empty_value: true - rename: + tag: rename_json_id field: json.Id target_field: aws.securityhub_findings_full_posture.id ignore_missing: true - set: + tag: set_event_id field: event.id copy_from: aws.securityhub_findings_full_posture.id ignore_failure: true - date: + tag: date_json_lastobservedat field: json.LastObservedAt if: ctx.json?.LastObservedAt != null && ctx.json?.LastObservedAt != '' target_field: aws.securityhub_findings_full_posture.last_observed_at @@ -742,6 +863,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_30 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: @@ -754,6 +876,7 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_31 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - set: @@ -762,6 +885,7 @@ processors: copy_from: aws.securityhub_findings_full_posture.processed_at ignore_empty_value: true - foreach: + tag: foreach_json_malware field: json.Malware processor: rename: @@ -771,6 +895,7 @@ processors: ignore_failure: true if: ctx.json?.Malware != null && ctx.json?.Malware instanceof List - foreach: + tag: foreach_json_malware_1 field: json.Malware processor: rename: @@ -780,6 +905,7 @@ processors: ignore_failure: true if: ctx.json?.Malware != null && ctx.json?.Malware instanceof List - foreach: + tag: foreach_json_malware_2 field: json.Malware processor: rename: @@ -789,6 +915,7 @@ processors: ignore_failure: true if: ctx.json?.Malware != null && ctx.json?.Malware instanceof List - foreach: + tag: foreach_json_malware_3 field: json.Malware processor: rename: @@ -798,18 +925,22 @@ processors: ignore_failure: true if: ctx.json?.Malware != null && ctx.json?.Malware instanceof List - rename: + tag: rename_json_malware field: json.Malware target_field: aws.securityhub_findings_full_posture.malware ignore_missing: true - rename: + tag: rename_json_network_destinationdomain field: json.Network.DestinationDomain target_field: aws.securityhub_findings_full_posture.network.destination.domain ignore_missing: true - set: + tag: set_destination_domain field: destination.domain copy_from: aws.securityhub_findings_full_posture.network.destination.domain ignore_failure: true - convert: + tag: convert_json_network_destinationipv4 field: json.Network.DestinationIpV4 target_field: aws.securityhub_findings_full_posture.network.destination.ip.v4 if: ctx.json?.Network?.DestinationIpV4 != '' @@ -817,15 +948,18 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_32 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_destination_ip field: destination.ip value: '{{{aws.securityhub_findings_full_posture.network.destination.ip.v4}}}' if: ctx.aws?.securityhub_findings_full_posture?.network?.destination?.ip?.v4 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_network_destinationipv6 field: json.Network.DestinationIpV6 target_field: aws.securityhub_findings_full_posture.network.destination.ip.v6 if: ctx.json?.Network?.DestinationIpV6 != '' @@ -833,15 +967,18 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_33 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_destination_ip_1 field: destination.ip value: '{{{aws.securityhub_findings_full_posture.network.destination.ip.v6}}}' if: ctx.aws?.securityhub_findings_full_posture?.network?.destination?.ip?.v6 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_network_destinationport field: json.Network.DestinationPort target_field: aws.securityhub_findings_full_posture.network.destination.port if: ctx.json?.Network?.DestinationPort != '' @@ -849,25 +986,31 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_34 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_destination_port field: destination.port copy_from: aws.securityhub_findings_full_posture.network.destination.port ignore_failure: true - rename: + tag: rename_json_network_direction field: json.Network.Direction target_field: aws.securityhub_findings_full_posture.network.direction ignore_missing: true - set: + tag: set_network_direction field: network.direction value: inbound if: "ctx.aws?.securityhub_findings_full_posture?.network?.direction == 'IN'" - set: + tag: set_network_direction_1 field: network.direction value: outbound if: "ctx.aws?.securityhub_findings_full_posture?.network?.direction == 'OUT'" - convert: + tag: convert_json_network_openportrange_begin field: json.Network.OpenPortRange.Begin target_field: aws.securityhub_findings_full_posture.network.open_port_range.begin if: ctx.json?.Network?.OpenPortRange?.Begin != '' @@ -875,9 +1018,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_35 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_network_openportrange_end field: json.Network.OpenPortRange.End target_field: aws.securityhub_findings_full_posture.network.open_port_range.end if: ctx.json?.Network?.OpenPortRange?.End != '' @@ -885,28 +1030,35 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_36 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_network_protocol field: json.Network.Protocol target_field: aws.securityhub_findings_full_posture.network.protocol ignore_missing: true - set: + tag: set_network_protocol field: network.protocol copy_from: aws.securityhub_findings_full_posture.network.protocol ignore_failure: true - lowercase: + tag: lowercase_network_protocol field: network.protocol ignore_missing: true - rename: + tag: rename_json_network_sourcedomain field: json.Network.SourceDomain target_field: aws.securityhub_findings_full_posture.network.source.domain ignore_missing: true - set: + tag: set_source_domain field: source.domain copy_from: aws.securityhub_findings_full_posture.network.source.domain ignore_failure: true - convert: + tag: convert_json_network_sourceipv4 field: json.Network.SourceIpV4 target_field: aws.securityhub_findings_full_posture.network.source.ip.v4 if: ctx.json?.Network?.SourceIpV4 != '' @@ -914,15 +1066,18 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_37 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_source_ip field: source.ip value: '{{{aws.securityhub_findings_full_posture.network.source.ip.v4}}}' if: ctx.aws?.securityhub_findings_full_posture?.network?.source?.ip?.v4 != null allow_duplicates: false ignore_failure: true - convert: + tag: convert_json_network_sourceipv6 field: json.Network.SourceIpV6 target_field: aws.securityhub_findings_full_posture.network.source.ip.v6 if: ctx.json?.Network?.SourceIpV6 != '' @@ -930,31 +1085,38 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_38 field: error.message value: '{{{_ingest.on_failure_message}}}' - append: + tag: append_source_ip_1 field: source.ip value: '{{{aws.securityhub_findings_full_posture.network.source.ip.v6}}}' if: ctx.aws?.securityhub_findings_full_posture?.network?.source?.ip?.v6 != null allow_duplicates: false ignore_failure: true - rename: + tag: rename_json_network_sourcemac field: json.Network.SourceMac target_field: aws.securityhub_findings_full_posture.network.source.mac ignore_missing: true - gsub: + tag: gsub_aws_securityhub_findings_full_posture_network_source_mac field: aws.securityhub_findings_full_posture.network.source.mac pattern: '[-:.]' replacement: '-' ignore_missing: true - uppercase: + tag: uppercase_aws_securityhub_findings_full_posture_network_source_mac field: aws.securityhub_findings_full_posture.network.source.mac ignore_missing: true - set: + tag: set_source_mac field: source.mac copy_from: aws.securityhub_findings_full_posture.network.source.mac ignore_failure: true - convert: + tag: convert_json_network_sourceport field: json.Network.SourcePort target_field: aws.securityhub_findings_full_posture.network.source.port if: ctx.json?.Network?.SourcePort != '' @@ -962,13 +1124,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_39 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_source_port field: source.port copy_from: aws.securityhub_findings_full_posture.network.source.port ignore_failure: true - foreach: + tag: foreach_json_networkpath field: json.NetworkPath processor: rename: @@ -978,6 +1143,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_1 field: json.NetworkPath processor: rename: @@ -987,6 +1153,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_2 field: json.NetworkPath processor: rename: @@ -996,6 +1163,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_3 field: json.NetworkPath processor: foreach: @@ -1008,6 +1176,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_40 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1015,6 +1184,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_4 field: json.NetworkPath processor: foreach: @@ -1027,6 +1197,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_41 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1034,6 +1205,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_5 field: json.NetworkPath processor: foreach: @@ -1049,6 +1221,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_6 field: json.NetworkPath processor: rename: @@ -1058,6 +1231,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_7 field: json.NetworkPath processor: rename: @@ -1067,6 +1241,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_8 field: json.NetworkPath processor: rename: @@ -1076,6 +1251,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_9 field: json.NetworkPath processor: foreach: @@ -1088,6 +1264,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_42 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1095,6 +1272,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_10 field: json.NetworkPath processor: foreach: @@ -1107,6 +1285,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_43 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1114,6 +1293,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_11 field: json.NetworkPath processor: foreach: @@ -1129,6 +1309,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_12 field: json.NetworkPath processor: rename: @@ -1138,6 +1319,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_13 field: json.NetworkPath processor: rename: @@ -1147,6 +1329,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_14 field: json.NetworkPath processor: foreach: @@ -1159,6 +1342,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_44 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1166,6 +1350,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_15 field: json.NetworkPath processor: foreach: @@ -1178,6 +1363,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_45 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1185,6 +1371,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_16 field: json.NetworkPath processor: foreach: @@ -1200,6 +1387,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_17 field: json.NetworkPath processor: rename: @@ -1209,6 +1397,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_18 field: json.NetworkPath processor: rename: @@ -1218,6 +1407,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_19 field: json.NetworkPath processor: rename: @@ -1227,6 +1417,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_20 field: json.NetworkPath processor: foreach: @@ -1239,6 +1430,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_46 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1247,6 +1439,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_21 field: json.NetworkPath processor: foreach: @@ -1259,6 +1452,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_47 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -1266,6 +1460,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_22 field: json.NetworkPath processor: foreach: @@ -1281,6 +1476,7 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - foreach: + tag: foreach_json_networkpath_23 field: json.NetworkPath processor: rename: @@ -1290,14 +1486,17 @@ processors: ignore_failure: true if: ctx.json?.NetworkPath != null && ctx.json?.NetworkPath instanceof List - rename: + tag: rename_json_networkpath field: json.NetworkPath target_field: aws.securityhub_findings_full_posture.network_path ignore_missing: true - rename: + tag: rename_json_note_text field: json.Note.Text target_field: aws.securityhub_findings_full_posture.note.text ignore_missing: true - date: + tag: date_json_note_updatedat field: json.Note.UpdatedAt if: ctx.json?.Note?.UpdatedAt != null && ctx.json?.Note?.UpdatedAt != '' target_field: aws.securityhub_findings_full_posture.note.updated_at @@ -1306,13 +1505,16 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_48 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_note_updatedby field: json.Note.UpdatedBy target_field: aws.securityhub_findings_full_posture.note.updated_by ignore_missing: true - convert: + tag: convert_json_patchsummary_failedcount field: json.PatchSummary.FailedCount target_field: aws.securityhub_findings_full_posture.patch_summary.failed.count if: ctx.json?.PatchSummary?.FailedCount != '' @@ -1320,13 +1522,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_49 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_patchsummary_id field: json.PatchSummary.Id target_field: aws.securityhub_findings_full_posture.patch_summary.id ignore_missing: true - convert: + tag: convert_json_patchsummary_installedcount field: json.PatchSummary.InstalledCount target_field: aws.securityhub_findings_full_posture.patch_summary.installed.count if: ctx.json?.PatchSummary?.InstalledCount != '' @@ -1334,9 +1539,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_50 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_patchsummary_installedothercount field: json.PatchSummary.InstalledOtherCount target_field: aws.securityhub_findings_full_posture.patch_summary.installed.other.count if: ctx.json?.PatchSummary?.InstalledOtherCount != '' @@ -1344,9 +1551,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_51 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_patchsummary_installedpendingreboot field: json.PatchSummary.InstalledPendingReboot target_field: aws.securityhub_findings_full_posture.patch_summary.installed.pending_reboot if: ctx.json?.PatchSummary?.InstalledPendingReboot != '' @@ -1354,9 +1563,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_52 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_patchsummary_installedrejectedcount field: json.PatchSummary.InstalledRejectedCount target_field: aws.securityhub_findings_full_posture.patch_summary.installed.rejected.count if: ctx.json?.PatchSummary?.InstalledRejectedCount != '' @@ -1364,9 +1575,11 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_53 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: + tag: convert_json_patchsummary_missingcount field: json.PatchSummary.MissingCount target_field: aws.securityhub_findings_full_posture.patch_summary.missing.count if: ctx.json?.PatchSummary?.MissingCount != '' @@ -1374,13 +1587,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_54 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_patchsummary_operation field: json.PatchSummary.Operation target_field: aws.securityhub_findings_full_posture.patch_summary.operation.type ignore_missing: true - date: + tag: date_json_patchsummary_operationendtime field: json.PatchSummary.OperationEndTime if: ctx.json?.PatchSummary?.OperationEndTime != null && ctx.json?.PatchSummary?.OperationEndTime != '' target_field: aws.securityhub_findings_full_posture.patch_summary.operation.end_time @@ -1389,9 +1605,11 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_55 field: error.message value: '{{{_ingest.on_failure_message}}}' - date: + tag: date_json_patchsummary_operationstarttime field: json.PatchSummary.OperationStartTime if: ctx.json?.PatchSummary?.OperationStartTime != null && ctx.json?.PatchSummary?.OperationStartTime != '' target_field: aws.securityhub_findings_full_posture.patch_summary.operation.start_time @@ -1400,13 +1618,16 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_56 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_patchsummary_rebootoption field: json.PatchSummary.RebootOption target_field: aws.securityhub_findings_full_posture.patch_summary.reboot_option ignore_missing: true - date: + tag: date_json_process_launchedat field: json.Process.LaunchedAt if: ctx.json?.Process?.LaunchedAt != null && ctx.json?.Process?.LaunchedAt != '' target_field: aws.securityhub_findings_full_posture.process.launched_at @@ -1415,21 +1636,26 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_57 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_process_start field: process.start copy_from: aws.securityhub_findings_full_posture.process.launched_at ignore_failure: true - rename: + tag: rename_json_process_name field: json.Process.Name target_field: aws.securityhub_findings_full_posture.process.name ignore_missing: true - set: + tag: set_process_name field: process.name copy_from: aws.securityhub_findings_full_posture.process.name ignore_failure: true - convert: + tag: convert_json_process_parentpid field: json.Process.ParentPid target_field: aws.securityhub_findings_full_posture.process.parent.pid if: ctx.json?.Process?.ParentPid != '' @@ -1437,21 +1663,26 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_58 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_process_parent_pid field: process.parent.pid copy_from: aws.securityhub_findings_full_posture.process.parent.pid ignore_failure: true - rename: + tag: rename_json_process_path field: json.Process.Path target_field: aws.securityhub_findings_full_posture.process.path ignore_missing: true - set: + tag: set_process_executable field: process.executable copy_from: aws.securityhub_findings_full_posture.process.path ignore_failure: true - convert: + tag: convert_json_process_pid field: json.Process.Pid target_field: aws.securityhub_findings_full_posture.process.pid if: ctx.json?.Process?.Pid != '' @@ -1459,13 +1690,16 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_59 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_process_pid field: process.pid copy_from: aws.securityhub_findings_full_posture.process.pid ignore_failure: true - date: + tag: date_json_process_terminatedat field: json.Process.TerminatedAt if: ctx.json?.Process?.TerminatedAt != null && ctx.json?.Process?.TerminatedAt != '' target_field: aws.securityhub_findings_full_posture.process.terminated_at @@ -1474,29 +1708,36 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSS'Z' on_failure: - append: + tag: append_error_message_60 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_process_end field: process.end copy_from: aws.securityhub_findings_full_posture.process.terminated_at ignore_failure: true - rename: + tag: rename_json_productarn field: json.ProductArn target_field: aws.securityhub_findings_full_posture.product.arn ignore_missing: true - rename: + tag: rename_json_productfields field: json.ProductFields target_field: aws.securityhub_findings_full_posture.product.fields ignore_missing: true - rename: + tag: rename_json_productname field: json.ProductName target_field: aws.securityhub_findings_full_posture.product.name ignore_missing: true - rename: + tag: rename_json_recordstate field: json.RecordState target_field: aws.securityhub_findings_full_posture.record_state ignore_missing: true - rename: + tag: rename_json_region field: json.Region target_field: aws.securityhub_findings_full_posture.region ignore_missing: true @@ -1506,6 +1747,7 @@ processors: copy_from: aws.securityhub_findings_full_posture.region ignore_empty_value: true - foreach: + tag: foreach_json_relatedfindings field: json.RelatedFindings processor: rename: @@ -1515,6 +1757,7 @@ processors: ignore_failure: true if: ctx.json?.RelatedFindings != null && ctx.json?.RelatedFindings instanceof List - foreach: + tag: foreach_json_relatedfindings_1 field: json.RelatedFindings processor: rename: @@ -1524,14 +1767,17 @@ processors: ignore_failure: true if: ctx.json?.RelatedFindings != null && ctx.json?.RelatedFindings instanceof List - rename: + tag: rename_json_relatedfindings field: json.RelatedFindings target_field: aws.securityhub_findings_full_posture.related_findings ignore_missing: true - rename: + tag: rename_json_remediation_recommendation_text field: json.Remediation.Recommendation.Text target_field: aws.securityhub_findings_full_posture.remediation.recommendation.text ignore_missing: true - rename: + tag: rename_json_remediation_recommendation_url field: json.Remediation.Recommendation.Url target_field: aws.securityhub_findings_full_posture.remediation.recommendation.url ignore_missing: true @@ -1547,6 +1793,7 @@ processors: if: ctx.aws?.securityhub_findings_full_posture?.remediation?.recommendation?.url != null && ctx.aws.securityhub_findings_full_posture.remediation.recommendation.text != null ignore_empty_value: true - rename: + tag: rename_json_resources field: json.Resources target_field: aws.securityhub_findings_full_posture.resources ignore_missing: true @@ -1916,6 +2163,7 @@ processors: } } - convert: + tag: convert_json_sample field: json.Sample target_field: aws.securityhub_findings_full_posture.sample if: ctx.json?.Sample != '' @@ -1923,17 +2171,21 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_61 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_schemaversion field: json.SchemaVersion target_field: aws.securityhub_findings_full_posture.schema.version ignore_missing: true - rename: + tag: rename_json_severity_label field: json.Severity.Label target_field: aws.securityhub_findings_full_posture.severity.label ignore_missing: true - convert: + tag: convert_json_severity_normalized field: json.Severity.Normalized target_field: aws.securityhub_findings_full_posture.severity.normalized if: ctx.json?.Severity?.Normalized != '' @@ -1941,6 +2193,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_62 field: error.message value: '{{{_ingest.on_failure_message}}}' - convert: @@ -1951,13 +2204,16 @@ processors: type: long on_failure: - append: + tag: append_error_message_63 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_severity_original field: json.Severity.Original target_field: aws.securityhub_findings_full_posture.severity.original ignore_missing: true - convert: + tag: convert_json_severity_product field: json.Severity.Product target_field: aws.securityhub_findings_full_posture.severity.product if: ctx.json?.Severity?.Product != '' @@ -1965,24 +2221,30 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_64 field: error.message value: '{{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_sourceurl field: json.SourceUrl target_field: aws.securityhub_findings_full_posture.source_url ignore_missing: true - uri_parts: + tag: uri_parts_aws_securityhub_findings_full_posture_source_url field: aws.securityhub_findings_full_posture.source_url if: ctx.aws?.securityhub_findings_full_posture?.source_url != '' && ctx.aws?.securityhub_findings_full_posture?.source_url != null on_failure: - append: + tag: append_error_message_65 field: error.message value: '{{{_ingest.on_failure_message}}}' - set: + tag: set_url_full field: url.full value: '{{{url.original}}}' ignore_failure: true - foreach: + tag: foreach_json_threatintelindicators field: json.ThreatIntelIndicators processor: rename: @@ -1992,6 +2254,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_1 field: json.ThreatIntelIndicators processor: date: @@ -2004,6 +2267,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_2 field: json.ThreatIntelIndicators processor: remove: @@ -2013,6 +2277,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_3 field: json.ThreatIntelIndicators processor: rename: @@ -2022,6 +2287,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_4 field: json.ThreatIntelIndicators processor: rename: @@ -2031,6 +2297,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_5 field: json.ThreatIntelIndicators processor: rename: @@ -2040,6 +2307,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_6 field: json.ThreatIntelIndicators processor: rename: @@ -2049,6 +2317,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - foreach: + tag: foreach_json_threatintelindicators_7 field: json.ThreatIntelIndicators processor: set: @@ -2058,6 +2327,7 @@ processors: ignore_failure: true if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - script: + tag: script description: Map box field ThreatIntelIndicator to ECS field threat.indicator.type lang: painless params: @@ -2101,10 +2371,12 @@ processors: } if: ctx.json?.ThreatIntelIndicators != null && ctx.json?.ThreatIntelIndicators instanceof List - rename: + tag: rename_json_threatintelindicators field: json.ThreatIntelIndicators target_field: aws.securityhub_findings_full_posture.threat_intel_indicators ignore_missing: true - rename: + tag: rename_json_title field: json.Title target_field: aws.securityhub_findings_full_posture.title ignore_missing: true @@ -2114,18 +2386,22 @@ processors: copy_from: aws.securityhub_findings_full_posture.title ignore_empty_value: true - rename: + tag: rename_json_types field: json.Types target_field: aws.securityhub_findings_full_posture.types ignore_missing: true - rename: + tag: rename_json_userdefinedfields field: json.UserDefinedFields target_field: aws.securityhub_findings_full_posture.user_defined_fields ignore_missing: true - rename: + tag: rename_json_verificationstate field: json.VerificationState target_field: aws.securityhub_findings_full_posture.verification_state ignore_missing: true - foreach: + tag: foreach_json_vulnerabilities field: json.Vulnerabilities processor: foreach: @@ -2145,6 +2421,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_1 field: json.Vulnerabilities processor: foreach: @@ -2164,6 +2441,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_2 field: json.Vulnerabilities processor: foreach: @@ -2178,6 +2456,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_3 field: json.Vulnerabilities processor: foreach: @@ -2190,6 +2469,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_66 field: error.message value: '{{{_ingest.on_failure_message}}}' ignore_failure: true @@ -2197,6 +2477,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_4 field: json.Vulnerabilities processor: foreach: @@ -2211,6 +2492,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_5 field: json.Vulnerabilities processor: foreach: @@ -2225,6 +2507,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_6 field: json.Vulnerabilities processor: foreach: @@ -2239,6 +2522,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_7 field: json.Vulnerabilities processor: foreach: @@ -2253,6 +2537,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_8 field: json.Vulnerabilities processor: foreach: @@ -2267,6 +2552,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_9 field: json.Vulnerabilities processor: foreach: @@ -2281,6 +2567,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_10 field: json.Vulnerabilities processor: rename: @@ -2290,6 +2577,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_11 field: json.Vulnerabilities processor: rename: @@ -2299,6 +2587,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_12 field: json.Vulnerabilities processor: set: @@ -2308,6 +2597,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_13 field: json.Vulnerabilities processor: rename: @@ -2317,6 +2607,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_14 field: json.Vulnerabilities processor: set: @@ -2326,6 +2617,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_15 field: json.Vulnerabilities processor: rename: @@ -2335,6 +2627,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_16 field: json.Vulnerabilities processor: rename: @@ -2344,6 +2637,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_17 field: json.Vulnerabilities processor: set: @@ -2353,6 +2647,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_18 field: json.Vulnerabilities processor: rename: @@ -2362,6 +2657,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_19 field: json.Vulnerabilities processor: date: @@ -2374,6 +2670,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_20 field: json.Vulnerabilities processor: rename: @@ -2383,6 +2680,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_21 field: json.Vulnerabilities processor: date: @@ -2395,6 +2693,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_22 field: json.Vulnerabilities processor: remove: @@ -2405,6 +2704,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_23 field: json.Vulnerabilities processor: foreach: @@ -2419,6 +2719,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_24 field: json.Vulnerabilities processor: foreach: @@ -2433,6 +2734,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_25 field: json.Vulnerabilities processor: foreach: @@ -2447,6 +2749,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_26 field: json.Vulnerabilities processor: foreach: @@ -2461,6 +2764,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_27 field: json.Vulnerabilities processor: foreach: @@ -2475,6 +2779,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_28 field: json.Vulnerabilities processor: foreach: @@ -2489,6 +2794,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_29 field: json.Vulnerabilities processor: foreach: @@ -2503,6 +2809,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_30 field: json.Vulnerabilities processor: foreach: @@ -2517,6 +2824,7 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - foreach: + tag: foreach_json_vulnerabilities_31 field: json.Vulnerabilities processor: rename: @@ -2526,34 +2834,41 @@ processors: ignore_failure: true if: ctx.json?.Vulnerabilities != null && ctx.json?.Vulnerabilities instanceof List - rename: + tag: rename_json_vulnerabilities field: json.Vulnerabilities target_field: aws.securityhub_findings_full_posture.vulnerabilities ignore_missing: true - rename: + tag: rename_json_workflow_status field: json.Workflow.Status target_field: aws.securityhub_findings_full_posture.workflow.status ignore_missing: true - rename: + tag: rename_json_workflowstate field: json.WorkflowState target_field: aws.securityhub_findings_full_posture.workflow.state ignore_missing: true - remove: + tag: remove field: - json ignore_missing: true - append: + tag: append_related_ip field: related.ip value: '{{{aws.securityhub_findings_full_posture.action.aws_api_call.remote_ip.ip.address_v4}}}' if: ctx.aws?.securityhub_findings_full_posture?.action?.aws_api_call?.remote_ip?.ip?.address_v4 != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_1 field: related.ip value: '{{{aws.securityhub_findings_full_posture.action.network_connection.remote_ip.ip.address_v4}}}' if: ctx.aws?.securityhub_findings_full_posture?.action?.network_connection?.remote_ip?.ip?.address_v4 != null allow_duplicates: false ignore_failure: true - foreach: + tag: foreach_aws_securityhub_findings_full_posture_action_port_probe_details field: aws.securityhub_findings_full_posture.action.port_probe.details processor: append: @@ -2564,6 +2879,7 @@ processors: ignore_failure: true if: ctx.aws?.securityhub_findings_full_posture?.action?.port_probe?.details != null && ctx.aws?.securityhub_findings_full_posture?.action?.port_probe?.details instanceof List - foreach: + tag: foreach_aws_securityhub_findings_full_posture_action_port_probe_details_1 field: aws.securityhub_findings_full_posture.action.port_probe.details processor: append: @@ -2574,30 +2890,35 @@ processors: ignore_failure: true if: ctx.aws?.securityhub_findings_full_posture?.action?.port_probe?.details != null && ctx.aws?.securityhub_findings_full_posture?.action?.port_probe?.details instanceof List - append: + tag: append_related_ip_2 field: related.ip value: '{{{aws.securityhub_findings_full_posture.network.destination.ip.v4}}}' if: ctx.aws?.securityhub_findings_full_posture?.network?.destination?.ip?.v4 != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_3 field: related.ip value: '{{{aws.securityhub_findings_full_posture.network.destination.ip.v6}}}' if: ctx.aws?.securityhub_findings_full_posture?.network?.destination?.ip?.v6 != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_4 field: related.ip value: '{{{aws.securityhub_findings_full_posture.network.source.ip.v4}}}' if: ctx.aws?.securityhub_findings_full_posture?.network?.source?.ip?.v4 != null allow_duplicates: false ignore_failure: true - append: + tag: append_related_ip_5 field: related.ip value: '{{{aws.securityhub_findings_full_posture.network.source.ip.v6}}}' if: ctx.aws?.securityhub_findings_full_posture?.network?.source?.ip?.v6 != null allow_duplicates: false ignore_failure: true - remove: + tag: remove_1 if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) field: - aws.securityhub_findings_full_posture.created_at @@ -2623,6 +2944,7 @@ processors: ignore_failure: true ignore_missing: true - foreach: + tag: foreach_aws_securityhub_findings_full_posture_threat_intel_indicators field: aws.securityhub_findings_full_posture.threat_intel_indicators processor: remove: @@ -2634,6 +2956,7 @@ processors: ignore_missing: true ignore_missing: true - foreach: + tag: foreach_aws_securityhub_findings_full_posture_vulnerabilities field: aws.securityhub_findings_full_posture.vulnerabilities processor: remove: @@ -2648,6 +2971,7 @@ processors: ignore_missing: true ignore_missing: true - script: + tag: script_1 description: Drops null/empty values recursively. lang: painless source: | @@ -2666,11 +2990,13 @@ processors: dropEmptyFields(ctx); on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message_67 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/securityhub_insights/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/securityhub_insights/elasticsearch/ingest_pipeline/default.yml index 3826e1b0a81..4affd486a29 100644 --- a/packages/aws/data_stream/securityhub_insights/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/securityhub_insights/elasticsearch/ingest_pipeline/default.yml @@ -13,12 +13,15 @@ processors: Removes the fields added by Agentless as metadata, as they can collide with ECS fields. - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - set: + tag: set_event_kind field: event.kind value: event - set: + tag: set_event_type field: event.type value: [info] - set: @@ -26,36 +29,44 @@ processors: value: AWS Security Hub CSPM tag: set_observer_vendor - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original field: event.original target_field: json ignore_failure: true - rename: + tag: rename_json_filters_awsaccountid field: json.Filters.AwsAccountId target_field: aws.securityhub_insights.filters.aws_account_id ignore_missing: true - rename: + tag: rename_json_filters_companyname field: json.Filters.CompanyName target_field: aws.securityhub_insights.filters.company.name ignore_missing: true - rename: + tag: rename_json_filters_compliancestatus field: json.Filters.ComplianceStatus target_field: aws.securityhub_insights.filters.compliance.status ignore_missing: true - rename: + tag: rename_json_filters_confidence field: json.Filters.Confidence target_field: aws.securityhub_insights.filters.confidence ignore_missing: true - foreach: + tag: foreach_json_filters_createdat field: json.Filters.CreatedAt processor: rename: @@ -65,6 +76,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.CreatedAt != null && ctx.json?.Filters?.CreatedAt instanceof List - foreach: + tag: foreach_json_filters_createdat_1 field: json.Filters.CreatedAt processor: rename: @@ -74,6 +86,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.CreatedAt != null && ctx.json?.Filters?.CreatedAt instanceof List - foreach: + tag: foreach_json_filters_createdat_2 field: json.Filters.CreatedAt processor: date: @@ -86,6 +99,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.CreatedAt != null && ctx.json?.Filters?.CreatedAt instanceof List - foreach: + tag: foreach_json_filters_createdat_3 field: json.Filters.CreatedAt processor: date: @@ -98,6 +112,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.CreatedAt != null && ctx.json?.Filters?.CreatedAt instanceof List - foreach: + tag: foreach_json_filters_createdat_4 field: json.Filters.CreatedAt processor: remove: @@ -108,46 +123,57 @@ processors: ignore_failure: true if: ctx.json?.Filters?.CreatedAt != null && ctx.json?.Filters?.CreatedAt instanceof List - rename: + tag: rename_json_filters_createdat field: json.Filters.CreatedAt target_field: aws.securityhub_insights.filters.created_at ignore_missing: true - rename: + tag: rename_json_filters_criticality field: json.Filters.Criticality target_field: aws.securityhub_insights.filters.criticality ignore_missing: true - rename: + tag: rename_json_filters_description field: json.Filters.Description target_field: aws.securityhub_insights.filters.description ignore_missing: true - rename: + tag: rename_json_filters_findingproviderfieldsconfidence field: json.Filters.FindingProviderFieldsConfidence target_field: aws.securityhub_insights.filters.finding_provider_fields.confidence ignore_missing: true - rename: + tag: rename_json_filters_findingproviderfieldscriticality field: json.Filters.FindingProviderFieldsCriticality target_field: aws.securityhub_insights.filters.finding_provider_fields.criticality ignore_missing: true - rename: + tag: rename_json_filters_findingproviderfieldsrelatedfindingsid field: json.Filters.FindingProviderFieldsRelatedFindingsId target_field: aws.securityhub_insights.filters.finding_provider_fields.related_findings.id ignore_missing: true - rename: + tag: rename_json_filters_findingproviderfieldsrelatedfindingsproductarn field: json.Filters.FindingProviderFieldsRelatedFindingsProductArn target_field: aws.securityhub_insights.filters.finding_provider_fields.related_findings.product.arn ignore_missing: true - rename: + tag: rename_json_filters_findingproviderfieldsseveritylabel field: json.Filters.FindingProviderFieldsSeverityLabel target_field: aws.securityhub_insights.filters.finding_provider_fields.severity.label ignore_missing: true - rename: + tag: rename_json_filters_findingproviderfieldsseverityoriginal field: json.Filters.FindingProviderFieldsSeverityOriginal target_field: aws.securityhub_insights.filters.finding_provider_fields.severity.original ignore_missing: true - rename: + tag: rename_json_filters_findingproviderfieldstypes field: json.Filters.FindingProviderFieldsTypes target_field: aws.securityhub_insights.filters.finding_provider_fields.types ignore_missing: true - foreach: + tag: foreach_json_filters_firstobservedat field: json.Filters.FirstObservedAt processor: rename: @@ -157,6 +183,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.FirstObservedAt != null && ctx.json?.Filters?.FirstObservedAt instanceof List - foreach: + tag: foreach_json_filters_firstobservedat_1 field: json.Filters.FirstObservedAt processor: rename: @@ -166,6 +193,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.FirstObservedAt != null && ctx.json?.Filters?.FirstObservedAt instanceof List - foreach: + tag: foreach_json_filters_firstobservedat_2 field: json.Filters.FirstObservedAt processor: date: @@ -178,6 +206,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.FirstObservedAt != null && ctx.json?.Filters?.FirstObservedAt instanceof List - foreach: + tag: foreach_json_filters_firstobservedat_3 field: json.Filters.FirstObservedAt processor: date: @@ -190,6 +219,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.FirstObservedAt != null && ctx.json?.Filters?.FirstObservedAt instanceof List - foreach: + tag: foreach_json_filters_firstobservedat_4 field: json.Filters.FirstObservedAt processor: remove: @@ -200,22 +230,27 @@ processors: ignore_failure: true if: ctx.json?.Filters?.FirstObservedAt != null && ctx.json?.Filters?.FirstObservedAt instanceof List - rename: + tag: rename_json_filters_firstobservedat field: json.Filters.FirstObservedAt target_field: aws.securityhub_insights.filters.first_observed_at ignore_missing: true - rename: + tag: rename_json_filters_generatorid field: json.Filters.GeneratorId target_field: aws.securityhub_insights.filters.generator.id ignore_missing: true - rename: + tag: rename_json_filters_id field: json.Filters.Id target_field: aws.securityhub_insights.filters.id ignore_missing: true - rename: + tag: rename_json_filters_keyword field: json.Filters.Keyword target_field: aws.securityhub_insights.filters.keyword ignore_missing: true - foreach: + tag: foreach_json_filters_lastobservedat field: json.Filters.LastObservedAt processor: rename: @@ -225,6 +260,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.LastObservedAt != null && ctx.json?.Filters?.LastObservedAt instanceof List - foreach: + tag: foreach_json_filters_lastobservedat_1 field: json.Filters.LastObservedAt processor: rename: @@ -234,6 +270,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.LastObservedAt != null && ctx.json?.Filters?.LastObservedAt instanceof List - foreach: + tag: foreach_json_filters_lastobservedat_2 field: json.Filters.LastObservedAt processor: date: @@ -246,6 +283,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.LastObservedAt != null && ctx.json?.Filters?.LastObservedAt instanceof List - foreach: + tag: foreach_json_filters_lastobservedat_3 field: json.Filters.LastObservedAt processor: date: @@ -258,6 +296,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.LastObservedAt != null && ctx.json?.Filters?.LastObservedAt instanceof List - foreach: + tag: foreach_json_filters_lastobservedat_4 field: json.Filters.LastObservedAt processor: remove: @@ -268,74 +307,92 @@ processors: ignore_failure: true if: ctx.json?.Filters?.LastObservedAt != null && ctx.json?.Filters?.LastObservedAt instanceof List - rename: + tag: rename_json_filters_lastobservedat field: json.Filters.LastObservedAt target_field: aws.securityhub_insights.filters.last_observed_at ignore_missing: true - rename: + tag: rename_json_filters_malwarename field: json.Filters.MalwareName target_field: aws.securityhub_insights.filters.malware.name ignore_missing: true - rename: + tag: rename_json_filters_malwarepath field: json.Filters.MalwarePath target_field: aws.securityhub_insights.filters.malware.path ignore_missing: true - rename: + tag: rename_json_filters_malwarestate field: json.Filters.MalwareState target_field: aws.securityhub_insights.filters.malware.state ignore_missing: true - rename: + tag: rename_json_filters_malwaretype field: json.Filters.MalwareType target_field: aws.securityhub_insights.filters.malware.type ignore_missing: true - rename: + tag: rename_json_filters_networkdestinationdomain field: json.Filters.NetworkDestinationDomain target_field: aws.securityhub_insights.filters.network.destination.domain ignore_missing: true - rename: + tag: rename_json_filters_networkdestinationipv4 field: json.Filters.NetworkDestinationIpV4 target_field: aws.securityhub_insights.filters.network.destination.ip.v4 ignore_missing: true - rename: + tag: rename_json_filters_networkdestinationipv6 field: json.Filters.NetworkDestinationIpV6 target_field: aws.securityhub_insights.filters.network.destination.ip.v6 ignore_missing: true - rename: + tag: rename_json_filters_networkdestinationport field: json.Filters.NetworkDestinationPort target_field: aws.securityhub_insights.filters.network.destination.port ignore_missing: true - rename: + tag: rename_json_filters_networkdirection field: json.Filters.NetworkDirection target_field: aws.securityhub_insights.filters.network.direction ignore_missing: true - rename: + tag: rename_json_filters_networkprotocol field: json.Filters.NetworkProtocol target_field: aws.securityhub_insights.filters.network.protocol ignore_missing: true - rename: + tag: rename_json_filters_networksourcedomain field: json.Filters.NetworkSourceDomain target_field: aws.securityhub_insights.filters.network.source.domain ignore_missing: true - rename: + tag: rename_json_filters_networksourceipv4 field: json.Filters.NetworkSourceIpV4 target_field: aws.securityhub_insights.filters.network.source.ip.v4 ignore_missing: true - rename: + tag: rename_json_filters_networksourceipv6 field: json.Filters.NetworkSourceIpV6 target_field: aws.securityhub_insights.filters.network.source.ip.v6 ignore_missing: true - rename: + tag: rename_json_filters_networksourcemac field: json.Filters.NetworkSourceMac target_field: aws.securityhub_insights.filters.network.source.mac ignore_missing: true - rename: + tag: rename_json_filters_networksourceport field: json.Filters.NetworkSourcePort target_field: aws.securityhub_insights.filters.network.source.port ignore_missing: true - rename: + tag: rename_json_filters_notetext field: json.Filters.NoteText target_field: aws.securityhub_insights.filters.note.text ignore_missing: true - foreach: + tag: foreach_json_filters_noteupdatedat field: json.Filters.NoteUpdatedAt processor: rename: @@ -345,6 +402,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.NoteUpdatedAt != null && ctx.json?.Filters?.NoteUpdatedAt instanceof List - foreach: + tag: foreach_json_filters_noteupdatedat_1 field: json.Filters.NoteUpdatedAt processor: rename: @@ -354,6 +412,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.NoteUpdatedAt != null && ctx.json?.Filters?.NoteUpdatedAt instanceof List - foreach: + tag: foreach_json_filters_noteupdatedat_2 field: json.Filters.NoteUpdatedAt processor: date: @@ -366,6 +425,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.NoteUpdatedAt != null && ctx.json?.Filters?.NoteUpdatedAt instanceof List - foreach: + tag: foreach_json_filters_noteupdatedat_3 field: json.Filters.NoteUpdatedAt processor: date: @@ -378,6 +438,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.NoteUpdatedAt != null && ctx.json?.Filters?.NoteUpdatedAt instanceof List - foreach: + tag: foreach_json_filters_noteupdatedat_4 field: json.Filters.NoteUpdatedAt processor: remove: @@ -388,14 +449,17 @@ processors: ignore_failure: true if: ctx.json?.Filters?.NoteUpdatedAt != null && ctx.json?.Filters?.NoteUpdatedAt instanceof List - rename: + tag: rename_json_filters_noteupdatedat field: json.Filters.NoteUpdatedAt target_field: aws.securityhub_insights.filters.note.updated_at ignore_missing: true - rename: + tag: rename_json_filters_noteupdatedby field: json.Filters.NoteUpdatedBy target_field: aws.securityhub_insights.filters.note.updated_by ignore_missing: true - foreach: + tag: foreach_json_filters_processlaunchedat field: json.Filters.ProcessLaunchedAt processor: rename: @@ -405,6 +469,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ProcessLaunchedAt != null && ctx.json?.Filters?.ProcessLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_processlaunchedat_1 field: json.Filters.ProcessLaunchedAt processor: rename: @@ -414,6 +479,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ProcessLaunchedAt != null && ctx.json?.Filters?.ProcessLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_processlaunchedat_2 field: json.Filters.ProcessLaunchedAt processor: date: @@ -426,6 +492,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ProcessLaunchedAt != null && ctx.json?.Filters?.ProcessLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_processlaunchedat_3 field: json.Filters.ProcessLaunchedAt processor: date: @@ -438,6 +505,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ProcessLaunchedAt != null && ctx.json?.Filters?.ProcessLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_processlaunchedat_4 field: json.Filters.ProcessLaunchedAt processor: remove: @@ -448,26 +516,32 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ProcessLaunchedAt != null && ctx.json?.Filters?.ProcessLaunchedAt instanceof List - rename: + tag: rename_json_filters_processlaunchedat field: json.Filters.ProcessLaunchedAt target_field: aws.securityhub_insights.filters.process.launched_at ignore_missing: true - rename: + tag: rename_json_filters_processname field: json.Filters.ProcessName target_field: aws.securityhub_insights.filters.process.name ignore_missing: true - rename: + tag: rename_json_filters_processparentpid field: json.Filters.ProcessParentPid target_field: aws.securityhub_insights.filters.process.parent.pid ignore_missing: true - rename: + tag: rename_json_filters_processpath field: json.Filters.ProcessPath target_field: aws.securityhub_insights.filters.process.path ignore_missing: true - rename: + tag: rename_json_filters_processpid field: json.Filters.ProcessPid target_field: aws.securityhub_insights.filters.process.pid ignore_missing: true - foreach: + tag: foreach_json_filters_processterminatedat field: json.Filters.ProcessTerminatedAt processor: rename: @@ -477,6 +551,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ProcessTerminatedAt != null && ctx.json?.Filters?.ProcessTerminatedAt instanceof List - foreach: + tag: foreach_json_filters_processterminatedat_1 field: json.Filters.ProcessTerminatedAt processor: rename: @@ -486,6 +561,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ProcessTerminatedAt != null && ctx.json?.Filters?.ProcessTerminatedAt instanceof List - foreach: + tag: foreach_json_filters_processterminatedat_2 field: json.Filters.ProcessTerminatedAt processor: date: @@ -498,6 +574,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ProcessTerminatedAt != null && ctx.json?.Filters?.ProcessTerminatedAt instanceof List - foreach: + tag: foreach_json_filters_processterminatedat_3 field: json.Filters.ProcessTerminatedAt processor: date: @@ -510,6 +587,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ProcessTerminatedAt != null && ctx.json?.Filters?.ProcessTerminatedAt instanceof List - foreach: + tag: foreach_json_filters_processterminatedat_4 field: json.Filters.ProcessTerminatedAt processor: remove: @@ -520,62 +598,77 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ProcessTerminatedAt != null && ctx.json?.Filters?.ProcessTerminatedAt instanceof List - rename: + tag: rename_json_filters_processterminatedat field: json.Filters.ProcessTerminatedAt target_field: aws.securityhub_insights.filters.process.terminated_at ignore_missing: true - rename: + tag: rename_json_filters_productarn field: json.Filters.ProductArn target_field: aws.securityhub_insights.filters.product.arn ignore_missing: true - rename: + tag: rename_json_filters_productfields field: json.Filters.ProductFields target_field: aws.securityhub_insights.filters.product.fields ignore_missing: true - rename: + tag: rename_json_filters_productname field: json.Filters.ProductName target_field: aws.securityhub_insights.filters.product.name ignore_missing: true - rename: + tag: rename_json_filters_recommendationtext field: json.Filters.RecommendationText target_field: aws.securityhub_insights.filters.recommendation_text ignore_missing: true - rename: + tag: rename_json_filters_recordstate field: json.Filters.RecordState target_field: aws.securityhub_insights.filters.record_state ignore_missing: true - rename: + tag: rename_json_filters_region field: json.Filters.Region target_field: aws.securityhub_insights.filters.region ignore_missing: true - rename: + tag: rename_json_filters_relatedfindingsid field: json.Filters.RelatedFindingsId target_field: aws.securityhub_insights.filters.related_findings.id ignore_missing: true - rename: + tag: rename_json_filters_relatedfindingsproductarn field: json.Filters.RelatedFindingsProductArn target_field: aws.securityhub_insights.filters.related_findings.product.arn ignore_missing: true - rename: + tag: rename_json_filters_resourceawsec2instanceiaminstanceprofilearn field: json.Filters.ResourceAwsEc2InstanceIamInstanceProfileArn target_field: aws.securityhub_insights.filters.resource.aws_ec2_instance.iam_instance_profile.arn ignore_missing: true - rename: + tag: rename_json_filters_resourceawsec2instanceimageid field: json.Filters.ResourceAwsEc2InstanceImageId target_field: aws.securityhub_insights.filters.resource.aws_ec2_instance.image.id ignore_missing: true - rename: + tag: rename_json_filters_resourceawsec2instanceipv4addresses field: json.Filters.ResourceAwsEc2InstanceIpV4Addresses target_field: aws.securityhub_insights.filters.resource.aws_ec2_instance.ip.v4_addresses ignore_missing: true - rename: + tag: rename_json_filters_resourceawsec2instanceipv6addresses field: json.Filters.ResourceAwsEc2InstanceIpV6Addresses target_field: aws.securityhub_insights.filters.resource.aws_ec2_instance.ip.v6_addresses ignore_missing: true - rename: + tag: rename_json_filters_resourceawsec2instancekeyname field: json.Filters.ResourceAwsEc2InstanceKeyName target_field: aws.securityhub_insights.filters.resource.aws_ec2_instance.key.name ignore_missing: true - foreach: + tag: foreach_json_filters_resourceawsec2instancelaunchedat field: json.Filters.ResourceAwsEc2InstanceLaunchedAt processor: rename: @@ -585,6 +678,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceAwsEc2InstanceLaunchedAt != null && ctx.json?.Filters?.ResourceAwsEc2InstanceLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_resourceawsec2instancelaunchedat_1 field: json.Filters.ResourceAwsEc2InstanceLaunchedAt processor: rename: @@ -594,6 +688,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceAwsEc2InstanceLaunchedAt != null && ctx.json?.Filters?.ResourceAwsEc2InstanceLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_resourceawsec2instancelaunchedat_2 field: json.Filters.ResourceAwsEc2InstanceLaunchedAt processor: date: @@ -606,6 +701,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceAwsEc2InstanceLaunchedAt != null && ctx.json?.Filters?.ResourceAwsEc2InstanceLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_resourceawsec2instancelaunchedat_3 field: json.Filters.ResourceAwsEc2InstanceLaunchedAt processor: date: @@ -618,6 +714,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceAwsEc2InstanceLaunchedAt != null && ctx.json?.Filters?.ResourceAwsEc2InstanceLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_resourceawsec2instancelaunchedat_4 field: json.Filters.ResourceAwsEc2InstanceLaunchedAt processor: remove: @@ -628,22 +725,27 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceAwsEc2InstanceLaunchedAt != null && ctx.json?.Filters?.ResourceAwsEc2InstanceLaunchedAt instanceof List - rename: + tag: rename_json_filters_resourceawsec2instancelaunchedat field: json.Filters.ResourceAwsEc2InstanceLaunchedAt target_field: aws.securityhub_insights.filters.resource.aws_ec2_instance.launched_at ignore_missing: true - rename: + tag: rename_json_filters_resourceawsec2instancesubnetid field: json.Filters.ResourceAwsEc2InstanceSubnetId target_field: aws.securityhub_insights.filters.resource.aws_ec2_instance.subnet.id ignore_missing: true - rename: + tag: rename_json_filters_resourceawsec2instancetype field: json.Filters.ResourceAwsEc2InstanceType target_field: aws.securityhub_insights.filters.resource.aws_ec2_instance.type ignore_missing: true - rename: + tag: rename_json_filters_resourceawsec2instancevpcid field: json.Filters.ResourceAwsEc2InstanceVpcId target_field: aws.securityhub_insights.filters.resource.aws_ec2_instance.vpc.id ignore_missing: true - foreach: + tag: foreach_json_filters_resourceawsiamaccesskeycreatedat field: json.Filters.ResourceAwsIamAccessKeyCreatedAt processor: rename: @@ -653,6 +755,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceAwsIamAccessKeyCreatedAt != null && ctx.json?.Filters?.ResourceAwsIamAccessKeyCreatedAt instanceof List - foreach: + tag: foreach_json_filters_resourceawsiamaccesskeycreatedat_1 field: json.Filters.ResourceAwsIamAccessKeyCreatedAt processor: rename: @@ -662,6 +765,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceAwsIamAccessKeyCreatedAt != null && ctx.json?.Filters?.ResourceAwsIamAccessKeyCreatedAt instanceof List - foreach: + tag: foreach_json_filters_resourceawsiamaccesskeycreatedat_2 field: json.Filters.ResourceAwsIamAccessKeyCreatedAt processor: date: @@ -674,6 +778,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceAwsIamAccessKeyCreatedAt != null && ctx.json?.Filters?.ResourceAwsIamAccessKeyCreatedAt instanceof List - foreach: + tag: foreach_json_filters_resourceawsiamaccesskeycreatedat_3 field: json.Filters.ResourceAwsIamAccessKeyCreatedAt processor: date: @@ -686,6 +791,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceAwsIamAccessKeyCreatedAt != null && ctx.json?.Filters?.ResourceAwsIamAccessKeyCreatedAt instanceof List - foreach: + tag: foreach_json_filters_resourceawsiamaccesskeycreatedat_4 field: json.Filters.ResourceAwsIamAccessKeyCreatedAt processor: remove: @@ -696,42 +802,52 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceAwsIamAccessKeyCreatedAt != null && ctx.json?.Filters?.ResourceAwsIamAccessKeyCreatedAt instanceof List - rename: + tag: rename_json_filters_resourceawsiamaccesskeycreatedat field: json.Filters.ResourceAwsIamAccessKeyCreatedAt target_field: aws.securityhub_insights.filters.resource.aws_iam_access_key.created_at ignore_missing: true - rename: + tag: rename_json_filters_resourceawsiamaccesskeyprincipalname field: json.Filters.ResourceAwsIamAccessKeyPrincipalName target_field: aws.securityhub_insights.filters.resource.aws_iam_access_key.principal.name ignore_missing: true - rename: + tag: rename_json_filters_resourceawsiamaccesskeystatus field: json.Filters.ResourceAwsIamAccessKeyStatus target_field: aws.securityhub_insights.filters.resource.aws_iam_access_key.status ignore_missing: true - rename: + tag: rename_json_filters_resourceawsiamaccesskeyusername field: json.Filters.ResourceAwsIamAccessKeyUserName target_field: aws.securityhub_insights.filters.resource.aws_iam_access_key.user.name ignore_missing: true - rename: + tag: rename_json_filters_resourceawsiamuserusername field: json.Filters.ResourceAwsIamUserUserName target_field: aws.securityhub_insights.filters.resource.aws_iam_user.user.name ignore_missing: true - rename: + tag: rename_json_filters_resourceawss3bucketownerid field: json.Filters.ResourceAwsS3BucketOwnerId target_field: aws.securityhub_insights.filters.resource.aws_s3_bucket.owner.id ignore_missing: true - rename: + tag: rename_json_filters_resourceawss3bucketownername field: json.Filters.ResourceAwsS3BucketOwnerName target_field: aws.securityhub_insights.filters.resource.aws_s3_bucket.owner.name ignore_missing: true - rename: + tag: rename_json_filters_resourcecontainerimageid field: json.Filters.ResourceContainerImageId target_field: aws.securityhub_insights.filters.resource.container.image.id ignore_missing: true - rename: + tag: rename_json_filters_resourcecontainerimagename field: json.Filters.ResourceContainerImageName target_field: aws.securityhub_insights.filters.resource.container.image.name ignore_missing: true - foreach: + tag: foreach_json_filters_resourcecontainerlaunchedat field: json.Filters.ResourceContainerLaunchedAt processor: rename: @@ -741,6 +857,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceContainerLaunchedAt != null && ctx.json?.Filters?.ResourceContainerLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_resourcecontainerlaunchedat_1 field: json.Filters.ResourceContainerLaunchedAt processor: rename: @@ -750,6 +867,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceContainerLaunchedAt != null && ctx.json?.Filters?.ResourceContainerLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_resourcecontainerlaunchedat_2 field: json.Filters.ResourceContainerLaunchedAt processor: date: @@ -762,6 +880,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceContainerLaunchedAt != null && ctx.json?.Filters?.ResourceContainerLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_resourcecontainerlaunchedat_3 field: json.Filters.ResourceContainerLaunchedAt processor: date: @@ -774,6 +893,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceContainerLaunchedAt != null && ctx.json?.Filters?.ResourceContainerLaunchedAt instanceof List - foreach: + tag: foreach_json_filters_resourcecontainerlaunchedat_4 field: json.Filters.ResourceContainerLaunchedAt processor: remove: @@ -784,62 +904,77 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ResourceContainerLaunchedAt != null && ctx.json?.Filters?.ResourceContainerLaunchedAt instanceof List - rename: + tag: rename_json_filters_resourcecontainerlaunchedat field: json.Filters.ResourceContainerLaunchedAt target_field: aws.securityhub_insights.filters.resource.container.launched_at ignore_missing: true - rename: + tag: rename_json_filters_resourcecontainername field: json.Filters.ResourceContainerName target_field: aws.securityhub_insights.filters.resource.container.name ignore_missing: true - rename: + tag: rename_json_filters_resourcedetailsother field: json.Filters.ResourceDetailsOther target_field: aws.securityhub_insights.filters.resource.details_other ignore_missing: true - rename: + tag: rename_json_filters_resourceid field: json.Filters.ResourceId target_field: aws.securityhub_insights.filters.resource.id ignore_missing: true - rename: + tag: rename_json_filters_resourcepartition field: json.Filters.ResourcePartition target_field: aws.securityhub_insights.filters.resource.partition ignore_missing: true - rename: + tag: rename_json_filters_resourceregion field: json.Filters.ResourceRegion target_field: aws.securityhub_insights.filters.resource.region ignore_missing: true - rename: + tag: rename_json_filters_resourcetags field: json.Filters.ResourceTags target_field: aws.securityhub_insights.filters.resource.tags ignore_missing: true - rename: + tag: rename_json_filters_resourcetype field: json.Filters.ResourceType target_field: aws.securityhub_insights.filters.resource.type ignore_missing: true - rename: + tag: rename_json_filters_sample field: json.Filters.Sample target_field: aws.securityhub_insights.filters.sample ignore_missing: true - rename: + tag: rename_json_filters_severitylabel field: json.Filters.SeverityLabel target_field: aws.securityhub_insights.filters.severity.label ignore_missing: true - rename: + tag: rename_json_filters_severitynormalized field: json.Filters.SeverityNormalized target_field: aws.securityhub_insights.filters.severity.normalized ignore_missing: true - rename: + tag: rename_json_filters_severityproduct field: json.Filters.SeverityProduct target_field: aws.securityhub_insights.filters.severity.product ignore_missing: true - rename: + tag: rename_json_filters_sourceurl field: json.Filters.SourceUrl target_field: aws.securityhub_insights.filters.source_url ignore_missing: true - rename: + tag: rename_json_filters_threatintelindicatorcategory field: json.Filters.ThreatIntelIndicatorCategory target_field: aws.securityhub_insights.filters.threat_intel_indicator.category ignore_missing: true - foreach: + tag: foreach_json_filters_threatintelindicatorlastobservedat field: json.Filters.ThreatIntelIndicatorLastObservedAt processor: rename: @@ -849,6 +984,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ThreatIntelIndicatorLastObservedAt != null && ctx.json?.Filters?.ThreatIntelIndicatorLastObservedAt instanceof List - foreach: + tag: foreach_json_filters_threatintelindicatorlastobservedat_1 field: json.Filters.ThreatIntelIndicatorLastObservedAt processor: rename: @@ -858,6 +994,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ThreatIntelIndicatorLastObservedAt != null && ctx.json?.Filters?.ThreatIntelIndicatorLastObservedAt instanceof List - foreach: + tag: foreach_json_filters_threatintelindicatorlastobservedat_2 field: json.Filters.ThreatIntelIndicatorLastObservedAt processor: date: @@ -870,6 +1007,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ThreatIntelIndicatorLastObservedAt != null && ctx.json?.Filters?.ThreatIntelIndicatorLastObservedAt instanceof List - foreach: + tag: foreach_json_filters_threatintelindicatorlastobservedat_3 field: json.Filters.ThreatIntelIndicatorLastObservedAt processor: date: @@ -882,6 +1020,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ThreatIntelIndicatorLastObservedAt != null && ctx.json?.Filters?.ThreatIntelIndicatorLastObservedAt instanceof List - foreach: + tag: foreach_json_filters_threatintelindicatorlastobservedat_4 field: json.Filters.ThreatIntelIndicatorLastObservedAt processor: remove: @@ -892,34 +1031,42 @@ processors: ignore_failure: true if: ctx.json?.Filters?.ThreatIntelIndicatorLastObservedAt != null && ctx.json?.Filters?.ThreatIntelIndicatorLastObservedAt instanceof List - rename: + tag: rename_json_filters_threatintelindicatorlastobservedat field: json.Filters.ThreatIntelIndicatorLastObservedAt target_field: aws.securityhub_insights.filters.threat_intel_indicator.last_observed_at ignore_missing: true - rename: + tag: rename_json_filters_threatintelindicatorsource field: json.Filters.ThreatIntelIndicatorSource target_field: aws.securityhub_insights.filters.threat_intel_indicator.source ignore_missing: true - rename: + tag: rename_json_filters_threatintelindicatorsourceurl field: json.Filters.ThreatIntelIndicatorSourceUrl target_field: aws.securityhub_insights.filters.threat_intel_indicator.source_url ignore_missing: true - rename: + tag: rename_json_filters_threatintelindicatortype field: json.Filters.ThreatIntelIndicatorType target_field: aws.securityhub_insights.filters.threat_intel_indicator.type ignore_missing: true - rename: + tag: rename_json_filters_threatintelindicatorvalue field: json.Filters.ThreatIntelIndicatorValue target_field: aws.securityhub_insights.filters.threat_intel_indicator.value ignore_missing: true - rename: + tag: rename_json_filters_title field: json.Filters.Title target_field: aws.securityhub_insights.filters.title ignore_missing: true - rename: + tag: rename_json_filters_type field: json.Filters.Type target_field: aws.securityhub_insights.filters.type ignore_missing: true - foreach: + tag: foreach_json_filters_updatedat field: json.Filters.UpdatedAt processor: rename: @@ -929,6 +1076,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.UpdatedAt != null && ctx.json?.Filters?.UpdatedAt instanceof List - foreach: + tag: foreach_json_filters_updatedat_1 field: json.Filters.UpdatedAt processor: rename: @@ -938,6 +1086,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.UpdatedAt != null && ctx.json?.Filters?.UpdatedAt instanceof List - foreach: + tag: foreach_json_filters_updatedat_2 field: json.Filters.UpdatedAt processor: date: @@ -950,6 +1099,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.UpdatedAt != null && ctx.json?.Filters?.UpdatedAt instanceof List - foreach: + tag: foreach_json_filters_updatedat_3 field: json.Filters.UpdatedAt processor: date: @@ -962,6 +1112,7 @@ processors: ignore_failure: true if: ctx.json?.Filters?.UpdatedAt != null && ctx.json?.Filters?.UpdatedAt instanceof List - foreach: + tag: foreach_json_filters_updatedat_4 field: json.Filters.UpdatedAt processor: remove: @@ -972,42 +1123,52 @@ processors: ignore_failure: true if: ctx.json?.Filters?.UpdatedAt != null && ctx.json?.Filters?.UpdatedAt instanceof List - rename: + tag: rename_json_filters_updatedat field: json.Filters.UpdatedAt target_field: aws.securityhub_insights.filters.updated_at ignore_missing: true - rename: + tag: rename_json_filters_userdefinedfields field: json.Filters.UserDefinedFields target_field: aws.securityhub_insights.filters.user_defined_fields ignore_missing: true - rename: + tag: rename_json_filters_verificationstate field: json.Filters.VerificationState target_field: aws.securityhub_insights.filters.verification.state ignore_missing: true - rename: + tag: rename_json_filters_workflowstate field: json.Filters.WorkflowState target_field: aws.securityhub_insights.filters.workflow.state ignore_missing: true - rename: + tag: rename_json_filters_workflowstatus field: json.Filters.WorkflowStatus target_field: aws.securityhub_insights.filters.workflow.status ignore_missing: true - rename: + tag: rename_json_groupbyattribute field: json.GroupByAttribute target_field: aws.securityhub_insights.group_by_attribute ignore_missing: true - rename: + tag: rename_json_insightarn field: json.InsightArn target_field: aws.securityhub_insights.insight_arn ignore_missing: true - rename: + tag: rename_json_name field: json.Name target_field: aws.securityhub_insights.name ignore_missing: true - remove: + tag: remove field: - json ignore_missing: true - script: + tag: script description: Drops null/empty values recursively. lang: painless source: | @@ -1026,11 +1187,13 @@ processors: dropEmptyFields(ctx); on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/sqs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/sqs/elasticsearch/ingest_pipeline/default.yml index 86d9664afa1..789043d40ad 100644 --- a/packages/aws/data_stream/sqs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/sqs/elasticsearch/ingest_pipeline/default.yml @@ -3,61 +3,75 @@ description: "Pipeline for SQS metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true - rename: + tag: rename_aws_sqs_metrics_approximateageofoldestmessage_max field: aws.sqs.metrics.ApproximateAgeOfOldestMessage.max target_field: aws.sqs.oldest_message_age.sec ignore_missing: true - rename: + tag: rename_aws_sqs_metrics_approximatenumberofmessagesdelayed_avg field: aws.sqs.metrics.ApproximateNumberOfMessagesDelayed.avg target_field: aws.sqs.messages.delayed ignore_missing: true - rename: + tag: rename_aws_sqs_metrics_approximatenumberofmessagesnotvisible_avg field: aws.sqs.metrics.ApproximateNumberOfMessagesNotVisible.avg target_field: aws.sqs.messages.not_visible ignore_missing: true - rename: + tag: rename_aws_sqs_metrics_approximatenumberofmessagesvisible_avg field: aws.sqs.metrics.ApproximateNumberOfMessagesVisible.avg target_field: aws.sqs.messages.visible ignore_missing: true - rename: + tag: rename_aws_sqs_metrics_numberofmessagesdeleted_sum field: aws.sqs.metrics.NumberOfMessagesDeleted.sum target_field: aws.sqs.messages.deleted ignore_missing: true - rename: + tag: rename_aws_sqs_metrics_numberofmessagesreceived_sum field: aws.sqs.metrics.NumberOfMessagesReceived.sum target_field: aws.sqs.messages.received ignore_missing: true - rename: + tag: rename_aws_sqs_metrics_numberofmessagessent_sum field: aws.sqs.metrics.NumberOfMessagesSent.sum target_field: aws.sqs.messages.sent ignore_missing: true - rename: + tag: rename_aws_sqs_metrics_numberofemptyreceives_sum field: aws.sqs.metrics.NumberOfEmptyReceives.sum target_field: aws.sqs.empty_receives ignore_missing: true - rename: + tag: rename_aws_sqs_metrics_sentmessagesize_avg field: aws.sqs.metrics.SentMessageSize.avg target_field: aws.sqs.sent_message_size.bytes ignore_missing: true - remove: + tag: remove field: - aws.sqs.metrics if: ctx.agent?.type != "firehose" ignore_missing: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/transitgateway/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/transitgateway/elasticsearch/ingest_pipeline/default.yml index 5bc398d2030..2ac9a0e07a5 100644 --- a/packages/aws/data_stream/transitgateway/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/transitgateway/elasticsearch/ingest_pipeline/default.yml @@ -3,20 +3,24 @@ description: "Pipeline for Transit Gateway metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/usage/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/usage/elasticsearch/ingest_pipeline/default.yml index 42af1dc1e0b..458fcb00c00 100644 --- a/packages/aws/data_stream/usage/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/usage/elasticsearch/ingest_pipeline/default.yml @@ -3,20 +3,24 @@ description: "Pipeline for Usage metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml index fce91b299af..e64e1b5991f 100644 --- a/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml @@ -3,98 +3,120 @@ description: Pipeline for AWS VPC Flow Logs processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - dot_expander: + tag: dot_expander_all field: "*" - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - set: + tag: set_event_type field: event.type value: [connection] - set: + tag: set_event_category field: event.category value: [network] - drop: + tag: drop if: 'ctx.event?.original.startsWith("version") || ctx.event?.original.startsWith("instance-id")' - dissect: + tag: dissect_event_original field: event.original pattern: '{"message":"%{event.original}"}' ignore_failure: true - script: + tag: script lang: painless if: ctx.event?.original != null source: >- ctx._temp_ = new HashMap(); ctx._temp_.message_token_count = ctx.event?.original.splitOnToken(" ").length; - dissect: + tag: dissect_event_original_1 field: event.original pattern: '%{aws.vpcflow.version} %{aws.vpcflow.account_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.log_status}' if: ctx?._temp_?.message_token_count == 14 - dissect: + tag: dissect_event_original_2 field: event.original pattern: '%{aws.vpcflow.instance_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr}' if: ctx?._temp_?.message_token_count == 6 - dissect: + tag: dissect_event_original_3 field: event.original pattern: '%{aws.vpcflow.version} %{aws.vpcflow.interface_id} %{aws.vpcflow.account_id} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.type} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{aws.vpcflow.action} %{aws.vpcflow.log_status}' if: ctx?._temp_?.message_token_count == 17 - dissect: + tag: dissect_event_original_4 field: event.original pattern: '%{aws.vpcflow.version} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.account_id} %{aws.vpcflow.type} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{aws.vpcflow.protocol} %{aws.vpcflow.bytes} %{aws.vpcflow.packets} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.log_status}' if: ctx?._temp_?.message_token_count == 21 - dissect: + tag: dissect_event_original_5 field: event.original pattern: '%{aws.vpcflow.version} %{aws.vpcflow.account_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.log_status} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.type} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{cloud.region} %{cloud.availability_zone} %{aws.vpcflow.sublocation.type} %{aws.vpcflow.sublocation.id} %{aws.vpcflow.pkt_src_service} %{aws.vpcflow.pkt_dst_service} %{network.direction} %{aws.vpcflow.traffic_path}' if: ctx?._temp_?.message_token_count == 29 - dissect: + tag: dissect_event_original_6 field: event.original description: default format for transit gateway vpc flow logs, covering versions v2 through v6. pattern: '%{aws.vpcflow.version} %{aws.vpcflow.resource_type} %{aws.vpcflow.account_id} %{aws.vpcflow.tgw_id} %{aws.vpcflow.tgw_attachment_id} %{aws.vpcflow.tgw_src_vpc_account_id} %{aws.vpcflow.tgw_dst_vpc_account_id} %{aws.vpcflow.tgw_src_vpc_id} %{aws.vpcflow.tgw_dst_vpc_id} %{aws.vpcflow.tgw_src_subnet_id} %{aws.vpcflow.tgw_dst_subnet_id} %{aws.vpcflow.tgw_src_eni} %{aws.vpcflow.tgw_dst_eni} %{aws.vpcflow.tgw_src_az_id} %{aws.vpcflow.tgw_dst_az_id} %{aws.vpcflow.tgw_pair_attachment_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.log_status} %{aws.vpcflow.type} %{aws.vpcflow.packets_lost_no_route} %{aws.vpcflow.packets_lost_blackhole} %{aws.vpcflow.packets_lost_mtu_exceeded} %{aws.vpcflow.packets_lost_ttl_expired} %{aws.vpcflow.tcp_flags} %{cloud.region} %{network.direction} %{aws.vpcflow.pkt_src_service} %{aws.vpcflow.pkt_dst_service}' if: ctx?._temp_?.message_token_count == 36 - dissect: + tag: dissect_event_original_7 field: event.original pattern: '%{aws.vpcflow.version} %{aws.vpcflow.account_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.log_status} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.type} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{cloud.region} %{cloud.availability_zone} %{aws.vpcflow.sublocation.type} %{aws.vpcflow.sublocation.id} %{aws.vpcflow.pkt_src_service} %{aws.vpcflow.pkt_dst_service} %{network.direction} %{aws.vpcflow.traffic_path} %{aws.vpcflow.ecs_cluster_arn} %{aws.vpcflow.ecs_cluster_name} %{aws.vpcflow.ecs_container_instance_arn} %{aws.vpcflow.ecs_container_instance_id} %{aws.vpcflow.ecs_container_id} %{aws.vpcflow.ecs_second_container_id} %{aws.vpcflow.ecs_service_name} %{aws.vpcflow.ecs_task_definition_arn} %{aws.vpcflow.ecs_task_arn} %{aws.vpcflow.ecs_task_id}' if: ctx?._temp_?.message_token_count == 39 - dissect: + tag: dissect_event_original_8 field: event.original pattern: '%{aws.vpcflow.version} %{aws.vpcflow.account_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.log_status} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.type} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{cloud.region} %{cloud.availability_zone} %{aws.vpcflow.sublocation.type} %{aws.vpcflow.sublocation.id} %{aws.vpcflow.pkt_src_service} %{aws.vpcflow.pkt_dst_service} %{network.direction} %{aws.vpcflow.traffic_path} %{aws.vpcflow.ecs_cluster_arn} %{aws.vpcflow.ecs_cluster_name} %{aws.vpcflow.ecs_container_instance_arn} %{aws.vpcflow.ecs_container_instance_id} %{aws.vpcflow.ecs_container_id} %{aws.vpcflow.ecs_second_container_id} %{aws.vpcflow.ecs_service_name} %{aws.vpcflow.ecs_task_definition_arn} %{aws.vpcflow.ecs_task_arn} %{aws.vpcflow.ecs_task_id} %{aws.vpcflow.reject_reason}' if: ctx?._temp_?.message_token_count == 40 # Convert Unix epoch to timestamp - date: + tag: date_aws_vpcflow_end field: aws.vpcflow.end target_field: '@timestamp' ignore_failure: true formats: - UNIX - date: + tag: date_aws_vpcflow_start field: aws.vpcflow.start target_field: event.start ignore_failure: true formats: - UNIX - date: + tag: date_aws_vpcflow_end_1 field: aws.vpcflow.end target_field: event.end ignore_failure: true formats: - UNIX - remove: + tag: remove field: - aws.vpcflow.start - aws.vpcflow.end ignore_missing: true - script: + tag: script_1 lang: painless ignore_failure: true if: ctx.aws != null @@ -120,86 +142,106 @@ processors: } handleMap(ctx.aws); - set: + tag: set_event_outcome field: event.outcome value: success if: ctx.aws?.vpcflow?.action == "ACCEPT" - set: + tag: set_event_outcome_1 field: event.outcome value: failure if: ctx.aws?.vpcflow?.action == "REJECT" - append: + tag: append_event_type field: event.type value: allowed if: ctx.aws?.vpcflow?.action == "ACCEPT" - append: + tag: append_event_type_1 field: event.type value: denied if: ctx.aws?.vpcflow?.action == "REJECT" - set: + tag: set_event_action field: event.action copy_from: aws.vpcflow.action ignore_empty_value: true - set: + tag: set_event_reason field: event.reason copy_from: aws.vpcflow.reject_reason ignore_empty_value: true - rename: + tag: rename_aws_vpcflow_srcaddr field: aws.vpcflow.srcaddr target_field: source.address ignore_missing: true - set: + tag: set_source_ip field: source.ip copy_from: source.address if: ctx.source?.address != null - convert: + tag: convert_aws_vpcflow_srcport field: aws.vpcflow.srcport target_field: source.port type: integer ignore_missing: true - rename: + tag: rename_aws_vpcflow_dstaddr field: aws.vpcflow.dstaddr target_field: destination.address ignore_missing: true - set: + tag: set_destination_ip field: destination.ip copy_from: destination.address if: ctx.destination?.address != null - convert: + tag: convert_aws_vpcflow_dstport field: aws.vpcflow.dstport target_field: destination.port type: integer ignore_missing: true - rename: + tag: rename_aws_vpcflow_protocol field: aws.vpcflow.protocol target_field: network.iana_number ignore_missing: true - convert: + tag: convert_aws_vpcflow_packets field: aws.vpcflow.packets target_field: source.packets type: long ignore_missing: true - convert: + tag: convert_aws_vpcflow_bytes field: aws.vpcflow.bytes target_field: source.bytes type: long ignore_missing: true - set: + tag: set_network_bytes field: network.bytes copy_from: source.bytes if: ctx.source?.bytes != null - set: + tag: set_network_packets field: network.packets copy_from: source.packets if: ctx.source?.packets != null - set: + tag: set_network_type field: network.type value: ipv4 if: 'ctx.source?.ip != null && ctx.source?.ip.contains(".")' - set: + tag: set_network_type_1 field: network.type value: ipv6 if: 'ctx.source?.ip != null && ctx.source?.ip.contains(":")' - script: + tag: script_2 lang: painless ignore_failure: true if: ctx?.network?.iana_number != null @@ -229,19 +271,23 @@ processors: ctx.network.transport = 'sctp'; } - community_id: + tag: community_id target_field: network.community_id ignore_failure: true # IP Geolocation Lookup - geoip: + tag: geoip_source_ip field: source.ip target_field: source.geo ignore_missing: true - geoip: + tag: geoip_destination_ip field: destination.ip target_field: destination.geo ignore_missing: true # IP Autonomous System (AS) Lookup - geoip: + tag: geoip_source_ip_1 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -250,6 +296,7 @@ processors: - organization_name ignore_missing: true - geoip: + tag: geoip_destination_ip_1 database_file: GeoLite2-ASN.mmdb field: destination.ip target_field: destination.as @@ -258,94 +305,117 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_source_as_organization_name field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true - rename: + tag: rename_destination_as_asn field: destination.as.asn target_field: destination.as.number ignore_missing: true - rename: + tag: rename_destination_as_organization_name field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true # Generate related.ip field - append: + tag: append_related_ip if: 'ctx.source?.ip != null && ctx.destination?.ip != null' field: related.ip value: ["{{source.ip}}", "{{destination.ip}}"] - set: + tag: set_cloud_provider field: cloud.provider value: aws - set: + tag: set_cloud_account_id if: ctx.aws?.vpcflow?.account_id != null field: cloud.account.id copy_from: aws.vpcflow.account_id - set: + tag: set_cloud_instance_id if: 'ctx.aws?.vpcflow?.instance_id != null && ctx.aws.vpcflow.instance_id != "-"' field: cloud.instance.id copy_from: aws.vpcflow.instance_id - convert: + tag: convert_aws_vpcflow_packets_lost_no_route field: aws.vpcflow.packets_lost_no_route type: long ignore_missing: true on_failure: - remove: + tag: remove_aws_vpcflow_packets_lost_no_route field: aws.vpcflow.packets_lost_no_route - convert: + tag: convert_aws_vpcflow_packets_lost_blackhole field: aws.vpcflow.packets_lost_blackhole type: long ignore_missing: true on_failure: - remove: + tag: remove_aws_vpcflow_packets_lost_blackhole field: aws.vpcflow.packets_lost_blackhole - convert: + tag: convert_aws_vpcflow_packets_lost_mtu_exceeded field: aws.vpcflow.packets_lost_mtu_exceeded type: long ignore_missing: true on_failure: - remove: + tag: remove_aws_vpcflow_packets_lost_mtu_exceeded field: aws.vpcflow.packets_lost_mtu_exceeded - convert: + tag: convert_aws_vpcflow_packets_lost_ttl_expired field: aws.vpcflow.packets_lost_ttl_expired type: long ignore_missing: true on_failure: - remove: + tag: remove_aws_vpcflow_packets_lost_ttl_expired field: aws.vpcflow.packets_lost_ttl_expired - set: + tag: set_orchestrator_cluster_id field: orchestrator.cluster.id copy_from: aws.vpcflow.ecs_cluster_arn if: ctx.aws?.vpcflow?.ecs_cluster_arn != '-' ignore_empty_value: true - set: + tag: set_orchestrator_cluster_name field: orchestrator.cluster.name copy_from: aws.vpcflow.ecs_cluster_name if: ctx.aws?.vpcflow?.ecs_cluster_name != '-' ignore_empty_value: true - set: + tag: set_orchestrator_resource_name field: orchestrator.resource.name copy_from: aws.vpcflow.ecs_container_instance_arn if: ctx.aws?.vpcflow?.ecs_container_instance_arn != '-' ignore_empty_value: true - set: + tag: set_orchestrator_resource_id field: orchestrator.resource.id copy_from: aws.vpcflow.ecs_container_instance_id if: ctx.aws?.vpcflow?.ecs_container_instance_id != '-' ignore_empty_value: true - set: + tag: set_service_name field: service.name copy_from: aws.vpcflow.ecs_service_name if: ctx.aws?.vpcflow?.ecs_service_name != '-' ignore_empty_value: true - set: + tag: set_event_kind field: event.kind value: event - script: + tag: script_3 lang: painless ignore_failure: true if: "ctx.aws?.vpcflow?.tcp_flags != null" @@ -376,6 +446,7 @@ processors: ctx.aws.vpcflow.tcp_flags_array.add('urg'); } - remove: + tag: remove_1 field: - _temp_ - aws.vpcflow.srcaddr @@ -394,11 +465,13 @@ processors: ignore_missing: true on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/vpn/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/vpn/elasticsearch/ingest_pipeline/default.yml index 6206ff91b95..b865053afd2 100644 --- a/packages/aws/data_stream/vpn/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/vpn/elasticsearch/ingest_pipeline/default.yml @@ -3,20 +3,24 @@ description: "Pipeline for VPN metrics" processors: - dot_expander: + tag: dot_expander_all field: "*" ignore_failure: true - set: + tag: set_cloud_account_name field: cloud.account.name copy_from: cloud.account.id override: false ignore_empty_value: true on_failure: - set: + tag: set_event_kind field: event.kind value: pipeline_error - append: + tag: append_error_message field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml index bd79968a154..119dd3eb6bc 100644 --- a/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml @@ -2,48 +2,59 @@ description: "Pipeline for WAF logs" processors: - set: + tag: set_ecs_version field: ecs.version value: '8.11.0' - set: + tag: set_event_category field: event.category value: ["web", "network"] - append: + tag: append_event_type field: event.type value: ["access"] - rename: + tag: rename_message field: message target_field: event.original ignore_missing: true if: 'ctx.event?.original == null' description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' - remove: + tag: remove_message field: message ignore_missing: true if: 'ctx.event?.original != null' description: 'The `message` field is no longer required if the document has an `event.original` field.' - json: + tag: json_event_original field: event.original target_field: json - date: + tag: date_json_timestamp field: json.timestamp target_field: '@timestamp' ignore_failure: true formats: - UNIX_MS - rename: + tag: rename_json_httprequest_clientip field: json.httpRequest.clientIp target_field: source.ip ignore_missing: true - geoip: + tag: geoip_source_ip field: source.ip target_field: source.geo ignore_missing: true - rename: + tag: rename_json_httprequest_country field: json.httpRequest.country target_field: source.geo.country_iso_code ignore_missing: true if: ctx.source?.geo?.country_iso_code == null - geoip: + tag: geoip_source_ip_1 database_file: GeoLite2-ASN.mmdb field: source.ip target_field: source.as @@ -52,15 +63,18 @@ processors: - organization_name ignore_missing: true - rename: + tag: rename_source_as_asn field: source.as.asn target_field: source.as.number ignore_missing: true - rename: + tag: rename_json_clientasn field: json.ClientASN target_field: source.as.number ignore_missing: true if: ctx?.source?.as?.number == null - rename: + tag: rename_source_as_organization_name field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true @@ -72,6 +86,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -83,6 +98,7 @@ processors: if: ctx.json?.captchaResponse?.solveTimestamp != null && ctx.json.captchaResponse.solveTimestamp != '' on_failure: - append: + tag: append_error_message_1 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -98,6 +114,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_2 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - date: @@ -109,6 +126,7 @@ processors: if: ctx.json?.challengeResponse?.solveTimestamp != null && ctx.json.challengeResponse.solveTimestamp != '' on_failure: - append: + tag: append_error_message_3 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: @@ -133,6 +151,7 @@ processors: target_field: url.registered_domain ignore_missing: true - rename: + tag: rename_json_httprequest_requestid field: json.httpRequest.requestId target_field: http.request.id ignore_missing: true @@ -149,6 +168,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_4 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -159,6 +179,7 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_5 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - convert: @@ -169,28 +190,35 @@ processors: ignore_missing: true on_failure: - append: + tag: append_error_message_6 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_httprequest_httpmethod field: json.httpRequest.httpMethod target_field: http.request.method ignore_missing: true - dissect: + tag: dissect_json_httprequest_httpversion field: json.httpRequest.httpVersion pattern: "%{network.protocol}/%{http.version}" ignore_failure: true - lowercase: + tag: lowercase_network_protocol field: network.protocol ignore_missing: true - set: + tag: set_network_transport field: network.transport value: tcp if: ctx?.network?.protocol != null && ctx?.network?.protocol == 'http' - rename: + tag: rename_json_httprequest_args field: json.httpRequest.args target_field: url.query ignore_missing: true - rename: + tag: rename_json_httprequest_uri field: json.httpRequest.uri target_field: url.path ignore_missing: true @@ -215,22 +243,27 @@ processors: target_field: aws.waf.oversize_fields ignore_missing: true - rename: + tag: rename_json_terminatingrulematchdetails field: json.terminatingRuleMatchDetails target_field: aws.waf.terminating_rule_match_details ignore_missing: true - rename: + tag: rename_json_rulegrouplist field: json.ruleGroupList target_field: aws.waf.rule_group_list ignore_missing: true - rename: + tag: rename_json_ratebasedrulelist field: json.rateBasedRuleList target_field: aws.waf.rate_based_rule_list ignore_missing: true - rename: + tag: rename_json_nonterminatingmatchingrules field: json.nonTerminatingMatchingRules target_field: aws.waf.non_terminating_matching_rules ignore_missing: true - script: + tag: script lang: painless source: >- if (ctx.json.httpRequest.headers != null) { @@ -253,53 +286,66 @@ processors: } on_failure: - append: + tag: append_error_message_7 field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - rename: + tag: rename_json_action field: json.action target_field: event.action ignore_missing: true - append: + tag: append_related_ip field: related.ip value: '{{source.ip}}' allow_duplicates: false if: ctx.source?.ip != null - set: + tag: set_cloud_provider field: cloud.provider value: aws - set: + tag: set_event_kind field: event.kind value: event - append: + tag: append_event_type_1 field: event.type value: allowed if: ctx.event.action == "ALLOW" - append: + tag: append_event_type_2 field: event.type value: denied if: ctx.event.action == "BLOCK" - rename: + tag: rename_json_webaclid field: json.webaclId target_field: aws.waf.arn ignore_missing: true - dissect: + tag: dissect_aws_waf_arn field: aws.waf.arn pattern: "arn:%{}:%{cloud.service.name}:%{cloud.region}:%{cloud.account.id}:%{aws.waf.id}" ignore_failure: true ignore_missing: true - rename: + tag: rename_json_terminatingruleid field: json.terminatingRuleId target_field: rule.id ignore_missing: true - rename: + tag: rename_json_terminatingruletype field: json.terminatingRuleType target_field: rule.ruleset ignore_missing: true - rename: + tag: rename_json_httpsourcename field: json.httpSourceName target_field: aws.waf.source.name ignore_missing: true - rename: + tag: rename_json_httpsourceid field: json.httpSourceId target_field: aws.waf.source.id ignore_missing: true @@ -308,9 +354,11 @@ processors: # Remove temporary fields # - remove: + tag: remove_json field: json ignore_missing: true - script: + tag: script_1 lang: painless description: This script processor iterates over the whole document to remove fields with null values. source: | @@ -336,11 +384,13 @@ processors: handleMap(ctx); on_failure: - set: + tag: set_event_kind_1 field: event.kind value: pipeline_error - append: + tag: append_error_message_8 field: error.message value: >- Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + {{{/_ingest.on_failure_processor_tag}}}in pipeline '{{{ _ingest.pipeline }}}' failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 40887ab276c..2ce76c26f98 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ -format_version: 3.4.0 +format_version: 3.6.0 name: aws title: AWS -version: 6.14.0 +version: 6.15.0 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration categories: @@ -21,7 +21,7 @@ conditions: elastic: subscription: basic kibana: - version: "^8.19.4 || ^9.2.1" + version: "^8.19.15 || ^9.4.0" screenshots: - src: /img/metricbeat-aws-overview.png title: metricbeat aws overview @@ -79,7 +79,22 @@ vars: multi: false required: false show_user: false + secret: true description: External ID to use when assuming a role in another account, see [the AWS documentation for use of external IDs](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html) + - name: assume_role_duration + type: duration + title: Assume Role Duration + multi: false + required: false + show_user: false + description: Duration of the assumed role session. + - name: assume_role_expiry_window + type: duration + title: Assume Role Expiry Window + multi: false + required: false + show_user: false + description: Amount of time before the assumed role session expires during which credentials are refreshed. - name: default_region type: text title: Default AWS Region @@ -95,6 +110,43 @@ vars: required: false show_user: false description: URL to proxy connections in the form of http\[s\]://:@: + - name: supports_cloud_connectors + type: bool + title: Supports Cloud Connectors + multi: false + required: false + show_user: false +var_groups: + - name: credential_type + required: true + title: Setup Access + selector_title: Preferred method + options: + - name: cloud_connectors + title: Cloud Connector + vars: [role_arn, external_id, supports_cloud_connectors] + hide_in_deployment_modes: [default] + provider: aws + iac_template_url: https://console.aws.amazon.com/cloudformation/home#/stacks/quickcreate?templateURL=https://elastic-cspm-cft.s3.eu-central-1.amazonaws.com/cloudformation-cloud-connectors-guardduty-9.4.0.yml¶m_ElasticResourceId=RESOURCE_ID + - name: direct_access_key + title: Direct Access Keys + vars: [access_key_id, secret_access_key] + - name: temporary_access_key + title: Temporary Access Keys + vars: [access_key_id, secret_access_key, session_token] + hide_in_deployment_modes: [agentless] + - name: assume_role + title: Assume Role + vars: [role_arn] + hide_in_deployment_modes: [agentless] + - name: assume_role_external_id + title: Assume Role with External ID + vars: [role_arn, external_id] + hide_in_deployment_modes: [agentless] + - name: shared_credentials + title: Shared Credentials + vars: [shared_credential_file, credential_profile_name] + hide_in_deployment_modes: [agentless] policy_templates: - name: awshealth title: AWS Health @@ -108,6 +160,8 @@ policy_templates: title: Collect AWS Health metrics (experimental) description: Collect AWS Health metrics (experimental). input_group: metrics + hide_in_var_group_options: + credential_type: [cloud_connectors] deployment_modes: default: enabled: true @@ -138,6 +192,8 @@ policy_templates: title: Collect billing metrics description: Collect billing metrics using AWS CloudWatch input_group: metrics + hide_in_var_group_options: + credential_type: [cloud_connectors] deployment_modes: default: enabled: true @@ -226,6 +282,8 @@ policy_templates: - type: cel title: Collect AWS Config logs via API description: Collecting AWS Config logs via API. + hide_in_var_group_options: + credential_type: [cloud_connectors] icons: - src: /img/logo-aws-config.svg title: AWS Config logo @@ -249,6 +307,8 @@ policy_templates: title: Collect dynamodb metrics description: Collect dynamodb metrics input_group: metrics + hide_in_var_group_options: + credential_type: [cloud_connectors] deployment_modes: default: enabled: true @@ -353,14 +413,20 @@ policy_templates: title: Collect ELB logs from S3 description: Collecting logs from ELB using aws-s3 input input_group: logs + hide_in_var_group_options: + credential_type: [cloud_connectors] - type: aws-cloudwatch title: Collect ELB logs from CloudWatch description: Collecting logs from ELB using aws-cloudwatch input input_group: logs + hide_in_var_group_options: + credential_type: [cloud_connectors] - type: aws/metrics title: Collect ELB metrics description: Collecting ELB metrics using AWS CloudWatch input_group: metrics + hide_in_var_group_options: + credential_type: [cloud_connectors] deployment_modes: default: enabled: true @@ -412,10 +478,14 @@ policy_templates: title: Collect Lambda metrics description: Collect Lambda metrics using AWS CloudWatch input_group: metrics + hide_in_var_group_options: + credential_type: [cloud_connectors] - type: aws-cloudwatch title: Collect lambda logs from CloudWatch description: Collecting AWS lambda logs using aws-cloudwatch input input_group: logs + hide_in_var_group_options: + credential_type: [cloud_connectors] deployment_modes: default: enabled: true @@ -513,6 +583,8 @@ policy_templates: title: Collect RDS metrics description: Collect RDS metrics using AWS CloudWatch input_group: metrics + hide_in_var_group_options: + credential_type: [cloud_connectors] deployment_modes: default: enabled: true @@ -599,6 +671,8 @@ policy_templates: title: Collect SNS metrics description: Collect SNS metrics using AWS CloudWatch input_group: metrics + hide_in_var_group_options: + credential_type: [cloud_connectors] deployment_modes: default: enabled: true @@ -629,6 +703,8 @@ policy_templates: title: Collect SQS metrics description: Collect SQS metrics using AWS CloudWatch input_group: metrics + hide_in_var_group_options: + credential_type: [cloud_connectors] deployment_modes: default: enabled: true @@ -659,6 +735,8 @@ policy_templates: title: Collect Transit Gateway metrics description: Collect Transit Gateway metrics using AWS CloudWatch input_group: metrics + hide_in_var_group_options: + credential_type: [cloud_connectors] deployment_modes: default: enabled: true @@ -866,6 +944,8 @@ policy_templates: - type: httpjson title: Collect AWS Security Hub CSPM logs via API description: Collecting AWS Security Hub CSPM logs via API. + hide_in_var_group_options: + credential_type: [cloud_connectors] screenshots: - src: /img/securityhub_cspm_findings_insights_dashboard.png title: Security Hub CSPM Findings and Insights dashboard screenshot @@ -901,6 +981,8 @@ policy_templates: - type: httpjson title: Collect Amazon Inspector logs via API description: Collecting Amazon Inspector logs via API. + hide_in_var_group_options: + credential_type: [cloud_connectors] screenshots: - src: /img/inspector-findings-overview-dashboard.png title: Inspector Findings Overview dashboard @@ -945,6 +1027,8 @@ policy_templates: - type: aws-s3 title: Collect Amazon GuardDuty logs via AWS S3 or SQS description: Collecting Amazon GuardDuty logs via AWS S3 or SQS input. + hide_in_var_group_options: + credential_type: [cloud_connectors] screenshots: - src: /img/guardduty-findings-overview.png title: GuardDuty Findings Overview dashboard screenshot