From b41db94752bf69b7d035738a0fe4fb9746b3d6d4 Mon Sep 17 00:00:00 2001 From: Leandro Maciel Date: Tue, 27 May 2025 00:19:20 -0300 Subject: [PATCH 1/6] feat: change the mapping of netskope.alerts.breach.date from double to date --- packages/netskope/changelog.yml | 5 +++++ .../pipeline/test-alerts.log-expected.json | 6 +++--- .../elasticsearch/ingest_pipeline/default.yml | 20 ++++++------------- .../data_stream/alerts/fields/fields.yml | 2 +- packages/netskope/docs/README.md | 2 +- packages/netskope/manifest.yml | 2 +- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/packages/netskope/changelog.yml b/packages/netskope/changelog.yml index 96b4332ddf8..2d6075adf3d 100644 --- a/packages/netskope/changelog.yml +++ b/packages/netskope/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.0.0" + changes: + - description: Change mapping of field `netskope.alerts.breach.date` from `double` to `date` + type: breaking-change + link: https://github.com/elastic/integrations/pull/TODO - version: "1.24.0" changes: - description: Add text multi-field to field netskope.alerts.breach.description diff --git a/packages/netskope/data_stream/alerts/_dev/test/pipeline/test-alerts.log-expected.json b/packages/netskope/data_stream/alerts/_dev/test/pipeline/test-alerts.log-expected.json index 2ff4fd1d200..e3b048af2ed 100644 --- a/packages/netskope/data_stream/alerts/_dev/test/pipeline/test-alerts.log-expected.json +++ b/packages/netskope/data_stream/alerts/_dev/test/pipeline/test-alerts.log-expected.json @@ -1227,7 +1227,7 @@ "type": "breach" }, "breach": { - "date": 1.6019424E9, + "date": "2020-10-06T00:00:00.000Z", "description": "In September 2020, a threat actor began sharing the millions of stolen credentials that were associated with a prominent Dark Web credentials service shut down by US federal authorites in August 2020. The stolen credentials represent hundreds of websites and hundreds of millions of users and their associated passwords affected by the illegal antics of the threat actor who managed the now defunct Dark Web forum. Users and companies from all over the world were affected by these various breaches. This file contains the download1.mios.com accounts dump.", "id": "bc6952df4c61b469cf4a47f17d0ea384", "score": 40 @@ -3478,7 +3478,7 @@ "type": "breach" }, "breach": { - "date": 1.6019424E9, + "date": "2020-10-06T00:00:00.000Z", "description": "In September 2020, a threat actor began sharing the millions of stolen credentials that were associated with a prominent Dark Web credentials service shut down by US federal authorites in August 2020. The stolen credentials represent hundreds of websites and hundreds of millions of users and their associated passwords affected by the illegal antics of the threat actor who managed the now defunct Dark Web forum. Users and companies from all over the world were affected by these various breaches. This file contains the download1.mios.com accounts dump.", "id": "bc6952df4c61b469cf4a47f17d0ea384", "score": 40 @@ -4133,7 +4133,7 @@ "category": "app" }, "breach": { - "date": 1.5054848E9, + "date": "2017-09-15T14:13:20.000Z", "description": "Test alert description", "id": "abcdefghd857e3cfbdb6d5704b48484", "score": 40, diff --git a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 32b528d8414..e6807559fb9 100644 --- a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -1558,21 +1558,13 @@ processors: Processor '{{{ _ingest.on_failure_processor_type }}}' {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' - - convert: - tag: convert_netskope_alerts_breach_date + - date: field: netskope.alerts.breach.date - type: double - ignore_missing: true - on_failure: - - remove: - field: netskope.alerts.breach.date - ignore_missing: true - - append: - field: error.message - value: >- - Processor '{{{ _ingest.on_failure_processor_type }}}' - {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' - {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' + formats: + - UNIX + - UNIX_MS + target_field: netskope.alerts.breach.date + ignore_failure: true - convert: tag: convert_netskope_alerts_malsite_latitude field: netskope.alerts.malsite.latitude diff --git a/packages/netskope/data_stream/alerts/fields/fields.yml b/packages/netskope/data_stream/alerts/fields/fields.yml index a8a2ff920dc..7cac24cbf37 100644 --- a/packages/netskope/data_stream/alerts/fields/fields.yml +++ b/packages/netskope/data_stream/alerts/fields/fields.yml @@ -225,7 +225,7 @@ description: | Breach description for compromised credentials. - name: date - type: double + type: date description: | Breach date for compromised credentials. - name: id diff --git a/packages/netskope/docs/README.md b/packages/netskope/docs/README.md index f611a4f52fd..ac09a820a69 100644 --- a/packages/netskope/docs/README.md +++ b/packages/netskope/docs/README.md @@ -114,7 +114,7 @@ Default port: _9021_ | netskope.alerts.audit.category | The subcategories in an application such as IAM, EC in AWS, login, token, file, etc., in case of Google. | keyword | | netskope.alerts.audit.type | The sub category in audit according to SaaS / IaaS apps. | keyword | | netskope.alerts.bin.timestamp | Applicable to only: Shared Credentials, Data Exfiltration, Bulk Anomaly types( Bulk Upload/Download/Delete) and Failed Login Anomaly type. Bin TimeStamp (is a window used that is used for certain types of anomalies - for breaking into several windows per day/hour). | long | -| netskope.alerts.breach.date | Breach date for compromised credentials. | double | +| netskope.alerts.breach.date | Breach date for compromised credentials. | date | | netskope.alerts.breach.description | Breach description for compromised credentials. | keyword | | netskope.alerts.breach.description.text | Multi-field of `netskope.alerts.breach.description`. | match_only_text | | netskope.alerts.breach.id | Breach ID for compromised credentials. | keyword | diff --git a/packages/netskope/manifest.yml b/packages/netskope/manifest.yml index d0c02b0cadc..82d82856523 100644 --- a/packages/netskope/manifest.yml +++ b/packages/netskope/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: netskope title: "Netskope" -version: "1.24.0" +version: "2.0.0" description: Collect logs from Netskope with Elastic Agent. type: integration categories: From a536116030259a419a6f53a30006c83122325098 Mon Sep 17 00:00:00 2001 From: Leandro Maciel Date: Tue, 27 May 2025 00:32:15 -0300 Subject: [PATCH 2/6] feat: add PR number, add conditiona to date processor --- packages/netskope/changelog.yml | 2 +- .../alerts/elasticsearch/ingest_pipeline/default.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/netskope/changelog.yml b/packages/netskope/changelog.yml index 2d6075adf3d..ee1679d8d51 100644 --- a/packages/netskope/changelog.yml +++ b/packages/netskope/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Change mapping of field `netskope.alerts.breach.date` from `double` to `date` type: breaking-change - link: https://github.com/elastic/integrations/pull/TODO + link: https://github.com/elastic/integrations/pull/14008 - version: "1.24.0" changes: - description: Add text multi-field to field netskope.alerts.breach.description diff --git a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index e6807559fb9..d57ce8386a9 100644 --- a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -1560,6 +1560,7 @@ processors: {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' - date: field: netskope.alerts.breach.date + if: ctx.netskope?.alerts?.breach?.date != null && ctx.netskope?.alerts?.breach?.date != '' formats: - UNIX - UNIX_MS From cd658c88d9621eb460bf11b7ece52bc2644ba340 Mon Sep 17 00:00:00 2001 From: Leandro Maciel Date: Tue, 27 May 2025 00:55:22 -0300 Subject: [PATCH 3/6] update packages/netskope/changelog.yml Co-authored-by: Dan Kortschak --- packages/netskope/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/netskope/changelog.yml b/packages/netskope/changelog.yml index ee1679d8d51..23b3eb933de 100644 --- a/packages/netskope/changelog.yml +++ b/packages/netskope/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top - version: "2.0.0" changes: - - description: Change mapping of field `netskope.alerts.breach.date` from `double` to `date` + - description: Change mapping of field `netskope.alerts.breach.date` from `double` to `date`. type: breaking-change link: https://github.com/elastic/integrations/pull/14008 - version: "1.24.0" From 2d0fdefe610c5944f125116435d445789f1f4e1e Mon Sep 17 00:00:00 2001 From: Leandro Maciel Date: Thu, 29 May 2025 13:21:13 -0300 Subject: [PATCH 4/6] feat: add on_failure and tag --- .../elasticsearch/ingest_pipeline/default.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index d57ce8386a9..0dccaa858f8 100644 --- a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -1559,13 +1559,24 @@ processors: {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' - date: + tag: date_netskope_alerts_breach_date field: netskope.alerts.breach.date - if: ctx.netskope?.alerts?.breach?.date != null && ctx.netskope?.alerts?.breach?.date != '' + if: ctx.netskope?.alerts?.breach?.date != null && ctx.netskope.alerts.breach.date != '' formats: - UNIX - UNIX_MS target_field: netskope.alerts.breach.date ignore_failure: true + on_failure: + - remove: + field: netskope.alerts.breach.date + ignore_missing: true + - append: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' - convert: tag: convert_netskope_alerts_malsite_latitude field: netskope.alerts.malsite.latitude From 107024815414ee47735ef6a524b888bab3999c34 Mon Sep 17 00:00:00 2001 From: Leandro Maciel Date: Fri, 30 May 2025 11:20:33 -0300 Subject: [PATCH 5/6] fix: remove UNIX_MS format as data is in seconds only --- .../data_stream/alerts/elasticsearch/ingest_pipeline/default.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 0dccaa858f8..3125630ed50 100644 --- a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -1564,7 +1564,6 @@ processors: if: ctx.netskope?.alerts?.breach?.date != null && ctx.netskope.alerts.breach.date != '' formats: - UNIX - - UNIX_MS target_field: netskope.alerts.breach.date ignore_failure: true on_failure: From b0f8700e0e7845b59ff66a8ee369cdb9ecc23d93 Mon Sep 17 00:00:00 2001 From: Leandro Maciel Date: Mon, 2 Jun 2025 10:44:32 -0300 Subject: [PATCH 6/6] Update packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml Co-authored-by: Dan Kortschak --- .../alerts/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 3125630ed50..6341780bade 100644 --- a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -1563,7 +1563,7 @@ processors: field: netskope.alerts.breach.date if: ctx.netskope?.alerts?.breach?.date != null && ctx.netskope.alerts.breach.date != '' formats: - - UNIX + - UNIX target_field: netskope.alerts.breach.date ignore_failure: true on_failure: