[Box Events] Remove non-ECS fields in ECS namespace

packages/box_events/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "2.16.0"
+  changes:
+    - description: |
+        Remove non-ECS fields that are collided with ECS namespace.
+        Following fields will no longer be available:
+        `related.location.lat`, `related.location.lon`, `related.description`, `related.indicator_type`.
+      type: breaking-change
+      link: https://github.com/elastic/integrations/pull/15947
 - version: "2.15.1"
   changes:
     - description: Fix the description of the interval setting.

packages/box_events/data_stream/events/_dev/test/pipeline/test-anomalous-download.log-expected.json

@@ -65,23 +65,11 @@
                 ]
             },
             "related": {
-                "description": [
-                    "Significant increase in download content week over week, 9200% (25.04 MB) more than last week 12 additional files downloaded week over week)"
-                ],
-                "indicator_type": [
-                    "file"
-                ],
                 "ip": [
                     "1.128.0.0",
                     "175.16.199.0",
                     "10.1.2.3"
                 ],
-                "location": [
-                    {
-                        "lat": 43.88,
-                        "lon": 125.3228
-                    }
-                ],
                 "user": [
                     "Unknown User",
                     "[email protected]",
@@ -211,22 +199,10 @@
                 ]
             },
             "related": {
-                "description": [
-                    "Significant increase in download content week over week, 9200% (25.04 MB) more than last week 12 additional files downloaded week over week)"
-                ],
-                "indicator_type": [
-                    "file"
-                ],
                 "ip": [
                     "175.16.199.0",
                     "10.1.2.3"
                 ],
-                "location": [
-                    {
-                        "lat": 43.88,
-                        "lon": 125.3228
-                    }
-                ],
                 "user": [
                     "Unknown User",
                     "[email protected]",

packages/box_events/data_stream/events/_dev/test/pipeline/test-invite.log-expected.json

@@ -62,12 +62,6 @@
                 "ip": [
                     "81.2.69.144"
                 ],
-                "location": [
-                    {
-                        "lat": 51.5142,
-                        "lon": -0.0931
-                    }
-                ],
                 "user": [
                     "[email protected]",
                     "Acting User",

packages/box_events/data_stream/events/_dev/test/pipeline/test-malicious-content.log-expected.json

@@ -80,22 +80,10 @@
                 ]
             },
             "related": {
-                "description": [
-                    "BadMalware, MalwareBot4000, malware.exe Detected by Box Shield from IP 67.43.156.0. This is a really bad file see https://some.link/xyz"
-                ],
-                "indicator_type": [
-                    "software"
-                ],
                 "ip": [
                     "67.43.156.0",
                     "10.1.2.3"
                 ],
-                "location": [
-                    {
-                        "lat": 27.5,
-                        "lon": 90.5
-                    }
-                ],
                 "user": [
                     "Unknown User",
                     "[email protected]",
@@ -220,12 +208,6 @@
                 ]
             },
             "related": {
-                "description": [
-                    "BadMalware, MalwareBot4000, malware.exe Detected by Box Shield from IP Unknown IP. This is a really bad file see https://some.link/xyz"
-                ],
-                "indicator_type": [
-                    "software"
-                ],
                 "user": [
                     "Unknown User",
                     "[email protected]",

packages/box_events/data_stream/events/_dev/test/pipeline/test-suspicious-locations.log-expected.json

@@ -38,26 +38,10 @@
                 ]
             },
             "related": {
-                "description": [
-                    "IP 81.2.69.144 was observed to Download file ABC/DEF/xyz.txt by Box Excel Online Previewer"
-                ],
-                "indicator_type": [
-                    "ipv4-addr"
-                ],
                 "ip": [
                     "81.2.69.144",
                     "67.43.156.0"
                 ],
-                "location": [
-                    {
-                        "lat": 51.5142,
-                        "lon": -0.0931
-                    },
-                    {
-                        "lat": 27.5,
-                        "lon": 90.5
-                    }
-                ],
                 "user": [
                     "Unknown User",
                     "[email protected]",
@@ -135,12 +119,6 @@
                 ]
             },
             "related": {
-                "description": [
-                    "IP Unknown IP was observed to Download file ABC/DEF/xyz.txt by Box Excel Online Previewer"
-                ],
-                "indicator_type": [
-                    "ipv4-addr"
-                ],
                 "user": [
                     "Unknown User",
                     "[email protected]",

packages/box_events/data_stream/events/_dev/test/pipeline/test-suspicious-sessions.log-expected.json

@@ -41,22 +41,10 @@
                 ]
             },
             "related": {
-                "description": [
-                    "IP 81.2.69.142 was observed to Set shared link expiration file ABC/DEF/xyz.txt by ServiceName"
-                ],
-                "indicator_type": [
-                    "user-account"
-                ],
                 "ip": [
                     "81.2.69.142",
                     "10.1.2.3"
                 ],
-                "location": [
-                    {
-                        "lat": 51.5142,
-                        "lon": -0.0931
-                    }
-                ],
                 "user": [
                     "Unknown User",
                     "[email protected]",
@@ -137,12 +125,6 @@
                 ]
             },
             "related": {
-                "description": [
-                    "IP Unknown IP was observed to Set shared link expiration file ABC/DEF/xyz.txt by ServiceName"
-                ],
-                "indicator_type": [
-                    "user-account"
-                ],
                 "user": [
                     "Unknown User",
                     "[email protected]",

packages/box_events/data_stream/events/elasticsearch/ingest_pipeline/default.yml

@@ -817,15 +817,6 @@ processors:
           if (ctx.related.ip == null) {
             ctx.related.ip = new ArrayList();
           }
-          if (ctx.related.description == null) {
-            ctx.related.description = new ArrayList();
-          }
-          if (ctx.related.location == null) {
-            ctx.related.location = new ArrayList();
-          }
-          if (ctx.related.indicator_type == null) {
-            ctx.related.indicator_type = new ArrayList();
-          }
         }
 #
   - script:
@@ -865,9 +856,6 @@ processors:
               indicator.indicator.put("type","user-account");    
               ctx.threat.enrichments.add(indicator);
               ctx.related.ip.add(geo.ip);
-              ctx.related.location.add(geo.location);
-              ctx.related.description.add(indicator.indicator.description);
-              ctx.related.indicator_type.add(indicator.indicator.type);
             }
           }
         }
@@ -911,9 +899,6 @@ processors:
           }  
           ctx.threat.enrichments.add(indicator);
           ctx.related.ip.add(geo.ip);
-          ctx.related.location.add(geo.location);
-          ctx.related.description.add(indicator.indicator.description);
-          ctx.related.indicator_type.add(indicator.indicator.type);
         }
   - script:
       description: Unpack Anomalous Download properties
@@ -944,8 +929,6 @@ processors:
             indicator.indicator.type = "file";
             ctx.threat.enrichments.add(indicator);
             ctx.related.ip.add(ip.ip);
-            ctx.related.description.add(indicator.indicator.description);
-            ctx.related.indicator_type.add(indicator.indicator.type);
           }
         }
   - script:
@@ -979,10 +962,8 @@ processors:
           ctx.threat.indicator.last_seen = ctx.box.additional_details.shield_alert.malware_info.last_seen;
           ctx.threat.indicator.reference = ctx.box.additional_details.shield_alert.malware_info.detail_link;
           ctx.related.ip.add(ctx.threat.indicator.ip);
-          ctx.related.description.add(ctx.threat.indicator.description);
         }
         ctx.threat.indicator.type = "software";
-        ctx.related.indicator_type.add(ctx.threat.indicator.type);
   - date:
       field: threat.indicator.first_seen
       tag: date_threat_indicator_first_seen
@@ -1070,27 +1051,12 @@ processors:
             .collect(Collectors.toList());
         }
         if (ctx.related != null) {
-          if (ctx.related.description != null) {
-            ctx.related.description = ctx.related.description.stream()
-              .distinct()
-              .collect(Collectors.toList());
-          }
-          if (ctx.related.indicator_type != null) {
-            ctx.related.indicator_type = ctx.related.indicator_type.stream()
-              .distinct()
-              .collect(Collectors.toList());
-          }
           if (ctx.related.ip != null) {
             ctx.related.ip = ctx.related.ip.stream()
               .filter(Objects::nonNull)
               .distinct()
               .collect(Collectors.toList());
           }
-          if (ctx.related.location != null) {
-            ctx.related.location = ctx.related.location.stream()
-              .distinct()
-              .collect(Collectors.toList());
-          }
         }
         if (ctx.box?.additional_details?.shield_alert?.alert_summary?.download_ips != null) {
           ctx.box.additional_details.shield_alert.alert_summary.download_ips = ctx.box.additional_details.shield_alert.alert_summary.download_ips.stream()
@@ -1244,42 +1210,6 @@ processors:
       field: threat.indicator.ip
       target_field: threat.indicator.geo
       ignore_missing: true
-  - script:
-      description: Initialize related.geo prior to geoip
-      lang: painless
-      source: |
-        if (ctx.related?.ip != null) {
-          ctx.related.geo = new ArrayList();
-          for (ip in ctx.related.ip) {
-            Map geo = new HashMap();
-            geo.put("ip",ip);
-            ctx.related.geo.add(geo);
-          }
-        }
-  - foreach:
-      field: related.geo
-      ignore_missing: true
-      processor:
-        geoip:
-          field: "_ingest._value.ip"
-          target_field: "_ingest._value.location"
-          ignore_missing: true
-  - script:
-      description: Collate derived locations to related.location
-      lang: painless
-      source: |
-        if (ctx.related?.geo != null) {
-          ctx.related.location = new ArrayList();
-          for (geo in ctx.related.geo) {
-            if (geo.location != null) {
-              ctx.related.location.add(geo.location.location);
-            }
-          }
-        }
-  - remove:
-      field: "related.geo"   
-      ignore_failure: true
-      ignore_missing: true      
 # AS look-ups on enrichments within Array
   - foreach:
       field: threat.enrichments

packages/box_events/data_stream/events/fields/agent.yml

@@ -23,29 +23,3 @@
       example: "stretch"
       description: >
         OS codename, if any.
-
-    - name: cpu.pct
-      type: scaled_float
-      description: >
-        Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1.
-
-    - name: network.in.bytes
-      type: long
-      description: >
-        The number of bytes received on all network interfaces by the host in a given period of time.
-
-    - name: network.in.packets
-      type: long
-      description: >
-        The number of packets received on all network interfaces by the host in a given period of time.
-
-    - name: network.out.bytes
-      type: long
-      description: >
-        The number of bytes sent out on all network interfaces by the host in a given period of time.
-
-    - name: network.out.packets
-      type: long
-      description: >
-        The number of packets sent out on all network interfaces by the host in a given period of time.
-

packages/box_events/data_stream/events/fields/fields.yml

@@ -451,22 +451,3 @@
         - name: user_name
           description: The name of the user that triggered the event.
           type: keyword
-- name: related
-  type: group
-  fields:
-    - name: location
-      description: Array of `location` derived from `related.ip`
-      type: group
-      fields:
-        - name: lat
-          description: Latitude coordinate
-          type: float
-        - name: lon
-          description: Longitude coordinate
-          type: float
-    - name: description
-      description: Array of `description` derived from `threat[.enrichments].indicator.description`
-      type: keyword
-    - name: indicator_type
-      description: Array of `indicator_type` derived from `threat[.enrichments].indicator.type`
-      type: keyword