[Netskope Alerts] Add text multi-field to netskope.alerts.breach.description field

packages/netskope/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.24.0"
+  changes:
+    - description: Correct the mappings of breach related fields and add parse for the date of the breach
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13977
 - version: "1.23.1"
   changes:
     - description: Ignore empty string values for some fields.

packages/netskope/data_stream/alerts/_dev/test/pipeline/test-alerts.log-expected.json

@@ -109,7 +109,6 @@
                     },
                     "type": "policy",
                     "url": {
-                        "extension": "com\\\\/open",
                         "original": "http:\\\\/\\\\/www.example.com\\\\/open?id=WLb5Mc7aPGx914gEyYNjJxTo32yjF8xKAcqIoN_klrGg",
                         "path": "\\\\/\\\\/www.example.com\\\\/open",
                         "query": "id=WLb5Mc7aPGx914gEyYNjJxTo32yjF8xKAcqIoN_klrGg",
@@ -283,7 +282,6 @@
                     },
                     "type": "DLP",
                     "url": {
-                        "extension": "com\\\\/open",
                         "original": "http:\\\\/\\\\/www.example.com\\\\/open?id=14WLYNjJxKgEyqIoNAcb57aPGx9_klcxTo3MyjF82rGg",
                         "path": "\\\\/\\\\/www.example.com\\\\/open",
                         "query": "id=14WLYNjJxKgEyqIoNAcb57aPGx9_klcxTo3MyjF82rGg",
@@ -470,7 +468,6 @@
                     },
                     "type": "quarantine",
                     "url": {
-                        "extension": "com\\\\/open",
                         "original": "https:\\\\/\\\\/www.example.com\\\\/open?id=o3MyjFxoNAcb514WLYNjJTI9_klcx82rGg7aPGxKgEyq",
                         "path": "\\\\/\\\\/www.example.com\\\\/open",
                         "query": "id=o3MyjFxoNAcb514WLYNjJTI9_klcx82rGg7aPGxKgEyq",
@@ -936,7 +933,6 @@
                     "page": {
                         "site": "examplesecuritycheck",
                         "url": {
-                            "extension": "com/tests/execute/9",
                             "original": "examplesecuritycheck.com/tests/execute/9",
                             "path": "examplesecuritycheck.com/tests/execute/9"
                         }
@@ -1227,7 +1223,7 @@
                         "type": "breach"
                     },
                     "breach": {
-                        "date": 1.6019424E9,
+                        "date": "2020-10-06T00:00:00.000Z",
                         "description": "In September 2020, a threat actor began sharing the millions of stolen credentials that were associated with a prominent Dark Web credentials service shut down by US federal authorites in August 2020. The stolen credentials represent hundreds of websites and hundreds of  millions of users and their associated passwords affected by the illegal antics of the threat actor who managed the now defunct Dark Web forum. Users and companies from all over the world were affected by these various breaches. This file contains the download1.mios.com accounts dump.",
                         "id": "bc6952df4c61b469cf4a47f17d0ea384",
                         "score": 40
@@ -1682,7 +1678,6 @@
                     "page": {
                         "site": "examplesecuritycheck",
                         "url": {
-                            "extension": "com/tests/execute/9",
                             "original": "examplesecuritycheck.com/tests/execute/9",
                             "path": "examplesecuritycheck.com/tests/execute/9"
                         }
@@ -2451,7 +2446,6 @@
                     "page": {
                         "site": "examplesecuritycheck",
                         "url": {
-                            "extension": "com/tests/execute/9",
                             "original": "examplesecuritycheck.com/tests/execute/9",
                             "path": "examplesecuritycheck.com/tests/execute/9"
                         }
@@ -3235,7 +3229,6 @@
                     },
                     "type": "DLP",
                     "url": {
-                        "extension": "com\\\\/open",
                         "original": "http:\\\\/\\\\/www.example.com\\\\/open?id=14WLYNjJxKgEyqIoNAcb57aPGx9_klcxTo3MyjF82rGg",
                         "path": "\\\\/\\\\/www.example.com\\\\/open",
                         "query": "id=14WLYNjJxKgEyqIoNAcb57aPGx9_klcxTo3MyjF82rGg",
@@ -3410,7 +3403,6 @@
                     },
                     "type": "DLP",
                     "url": {
-                        "extension": "com\\\\/open",
                         "original": "http:\\\\/\\\\/www.example.com\\\\/open?id=14WLYNjJxKgEyqIoNAcb57aPGx9_klcxTo3MyjF82rGg",
                         "path": "\\\\/\\\\/www.example.com\\\\/open",
                         "query": "id=14WLYNjJxKgEyqIoNAcb57aPGx9_klcxTo3MyjF82rGg",
@@ -3478,7 +3470,7 @@
                         "type": "breach"
                     },
                     "breach": {
-                        "date": 1.6019424E9,
+                        "date": "2020-10-06T00:00:00.000Z",
                         "description": "In September 2020, a threat actor began sharing the millions of stolen credentials that were associated with a prominent Dark Web credentials service shut down by US federal authorites in August 2020. The stolen credentials represent hundreds of websites and hundreds of  millions of users and their associated passwords affected by the illegal antics of the threat actor who managed the now defunct Dark Web forum. Users and companies from all over the world were affected by these various breaches. This file contains the download1.mios.com accounts dump.",
                         "id": "bc6952df4c61b469cf4a47f17d0ea384",
                         "score": 40
@@ -3629,7 +3621,6 @@
                     "page": {
                         "site": "examplesecuritycheck",
                         "url": {
-                            "extension": "com/tests/execute/9",
                             "original": "examplesecuritycheck.com/tests/execute/9",
                             "path": "examplesecuritycheck.com/tests/execute/9"
                         }
@@ -4133,7 +4124,7 @@
                         "category": "app"
                     },
                     "breach": {
-                        "date": 1.5054848E9,
+                        "date": "2017-09-15T14:13:20.000Z",
                         "description": "Test alert description",
                         "id": "abcdefghd857e3cfbdb6d5704b48484",
                         "score": 40,
@@ -4248,7 +4239,6 @@
                     },
                     "type": "policy",
                     "url": {
-                        "extension": "com/",
                         "original": "www.example.com/",
                         "path": "www.example.com/"
                     }

packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml

@@ -1573,6 +1573,13 @@ processors:
               Processor '{{{ _ingest.on_failure_processor_type }}}'
               {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
               {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
+  - date:
+      field: netskope.alerts.breach.date
+      formats:
+      - UNIX
+      - UNIX_MS
+      target_field: netskope.alerts.breach.date
+      ignore_failure: true
   - convert:
       tag: convert_netskope_alerts_malsite_latitude
       field: netskope.alerts.malsite.latitude

packages/netskope/data_stream/alerts/fields/fields.yml

@@ -219,10 +219,13 @@
       fields:
         - name: description
           type: keyword
+          multi_fields:
+            - name: text
+              type: match_only_text
           description: |
-            N/A
+            Breach description for compromised credentials.
         - name: date
-          type: double
+          type: date
           description: |
             Breach date for compromised credentials.
         - name: id

packages/netskope/data_stream/events/_dev/test/pipeline/test-events.log-expected.json

@@ -535,7 +535,7 @@
                 "os": {
                     "name": "Ubuntu"
                 },
-                "version": "95.0."
+                "version": "95.0"
             }
         },
         {
@@ -630,7 +630,6 @@
                     },
                     "type": "connection",
                     "url": {
-                        "extension": "com",
                         "original": "some.example.com",
                         "path": "some.example.com"
                     },
@@ -1234,7 +1233,7 @@
                 "os": {
                     "name": "Ubuntu"
                 },
-                "version": "95.0."
+                "version": "95.0"
             }
         },
         {
@@ -1329,7 +1328,6 @@
                     },
                     "type": "connection",
                     "url": {
-                        "extension": "com",
                         "original": "some.example.com",
                         "path": "some.example.com"
                     },
@@ -1633,7 +1631,6 @@
                     },
                     "type": "connection",
                     "url": {
-                        "extension": "com",
                         "original": "some.example.com",
                         "path": "some.example.com"
                     },
@@ -2201,7 +2198,6 @@
                     },
                     "type": "connection",
                     "url": {
-                        "extension": "com",
                         "original": "example.com",
                         "path": "example.com"
                     },
@@ -2315,7 +2311,6 @@
                     },
                     "type": "connection",
                     "url": {
-                        "extension": "com",
                         "original": "example.com",
                         "path": "example.com"
                     },
@@ -2421,4 +2416,4 @@
             }
         }
     ]
-}
+}

packages/netskope/docs/README.md

@@ -114,8 +114,9 @@ Default port: _9021_
 | netskope.alerts.audit.category | The subcategories in an application such as IAM, EC in AWS, login, token, file, etc., in case of Google. | keyword |
 | netskope.alerts.audit.type | The sub category in audit according to SaaS / IaaS apps. | keyword |
 | netskope.alerts.bin.timestamp | Applicable to only: Shared Credentials, Data Exfiltration, Bulk Anomaly types( Bulk Upload/Download/Delete) and Failed Login Anomaly type. Bin TimeStamp (is a window used that is used for certain types of anomalies - for breaking into several windows per day/hour). | long |
-| netskope.alerts.breach.date | Breach date for compromised credentials. | double |
-| netskope.alerts.breach.description | N/A | keyword |
+| netskope.alerts.breach.date | Breach date for compromised credentials. | date |
+| netskope.alerts.breach.description | Breach description for compromised credentials. | keyword |
+| netskope.alerts.breach.description.text | Multi-field of `netskope.alerts.breach.description`. | match_only_text |
 | netskope.alerts.breach.id | Breach ID for compromised credentials. | keyword |
 | netskope.alerts.breach.media_references | Media references of breach. | keyword |
 | netskope.alerts.breach.score | Breach score for compromised credentials. | long |

packages/netskope/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: netskope
 title: "Netskope"
-version: "1.23.1"
+version: "1.24.0"
 description: Collect logs from Netskope with Elastic Agent.
 type: integration
 categories: