Update checkpoint to ECS 8.2

packages/checkpoint/_dev/build/build.yml

@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@8.0
+    reference: git@8.2

packages/checkpoint/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.4.0"
+  changes:
+    - description: Update to ECS 8.2 to use new email field set.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2803
 - version: "1.3.6"
   changes:
     - description: Fixed parsing error when logs have trailing spaces

packages/checkpoint/data_stream/firewall/_dev/test/pipeline/test-checkpoint-with-time.log-expected.json

@@ -13,7 +13,7 @@
                 "port": 514
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -91,7 +91,7 @@
                 "port": 80
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Drop",

packages/checkpoint/data_stream/firewall/_dev/test/pipeline/test-checkpoint.log-expected.json

@@ -6,7 +6,7 @@
                 "sys_message": "The eth0 interface is not protected by the anti-spoofing feature. Your network may be at risk"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "category": [
@@ -41,7 +41,7 @@
                 "sys_message": "installed Standard"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "category": [
@@ -83,7 +83,7 @@
                 "port": 53
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -166,7 +166,7 @@
                 "port": 443
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -238,7 +238,7 @@
                 "port": 53
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -302,7 +302,7 @@
                 "status": "Finished"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "category": [
@@ -353,7 +353,7 @@
                 "port": 80
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -439,7 +439,7 @@
                 "port": 53
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -498,7 +498,7 @@
                 "status": "Started"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "category": [
@@ -549,7 +549,7 @@
                 "port": 80
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -621,7 +621,7 @@
                 "port": 53
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -685,7 +685,7 @@
                 "status": "Finished"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "category": [
@@ -722,7 +722,7 @@
                 "port": 514
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -786,7 +786,7 @@
                 "update_status": "updated"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "category": [
@@ -819,7 +819,7 @@
                 "update_status": "updated"
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "category": [
@@ -857,7 +857,7 @@
                 "port": 138
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -937,7 +937,7 @@
                 "port": 80
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Drop",
@@ -992,7 +992,7 @@
                 "port": 514
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -1061,7 +1061,7 @@
                 "port": 137
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -1130,7 +1130,7 @@
                 "port": 22
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",
@@ -1199,7 +1199,7 @@
                 "port": 514
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",

packages/checkpoint/data_stream/firewall/_dev/test/pipeline/test-trailing-space.log-expected.json

@@ -13,7 +13,7 @@
                 "port": 53
             },
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.2.0"
             },
             "event": {
                 "action": "Accept",

packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml

@@ -3,7 +3,7 @@ description: Pipeline for parsing checkpoint firewall logs
 processors:
   - set:
       field: ecs.version
-      value: '8.0.0'
+      value: '8.2.0'
   - rename:
       field: message
       target_field: event.original
@@ -171,10 +171,6 @@ processors:
       field: checkpoint.mac_source_address
       target_field: source.mac
       ignore_missing: true
-  - rename:
-      field: checkpoint.from
-      target_field: source.user.email
-      ignore_missing: true
   - rename:
       field: checkpoint.src_machine_name
       target_field: source.domain
@@ -268,10 +264,54 @@ processors:
       field: related.hash
       value: "{{checkpoint.file_sha256}}"
       if: ctx.checkpoint?.file_sha256 != null
+  - rename:
+      field: checkpoint.from
+      target_field: source.user.email
+      ignore_missing: true
   - rename:
       field: checkpoint.to
       target_field: destination.user.email
       ignore_missing: true
+  - set:
+      field: email.from.address
+      value: ["{{{destination.user.email}}}"]
+      if: "ctx?.destination?.user?.email != null"
+  - set:
+      field: email.to.address
+      value: ["{{{destination.user.email}}}"]
+      if: "ctx?.destination?.user?.email != null"
+  - append:
+      field: email.from.address
+      value: "{{{checkpoint.mime_from}}}"
+      if: "ctx?.checkpoint?.mime_from != null"
+  - append:
+      field: email.to.address
+      value: "{{{checkpoint.mime_to}}}"
+      if: "ctx?.checkpoint?.mime_to != null"
+  - set:
+      field: email.subject
+      copy_from: checkpoint.email_subject
+      if: "ctx?.checkpoint?.email_subject != null"
+  - append:
+      field: email.bcc.address
+      value: "{{{checkpoint.bcc}}}"
+      if: "ctx?.checkpoint?.bcc != null"
+  - append:
+      field: email.cc.address
+      value: "{{{checkpoint.cc}}}"
+      if: "ctx?.checkpoint?.cc != null"
+  - set:
+      field: email.delivery_timestamp
+      copy_from: checkpoint.delivery_time
+      if: "ctx?.checkpoint?.delivery_time != null"
+  - set:
+      field: email.message_id
+      copy_from: checkpoint.email_message_id
+      if: "ctx?.checkpoint?.email_message_id != null"
+  - set:
+      field: email.local_id
+      copy_from: checkpoint.email_queue_id
+      if: "ctx?.checkpoint?.email_queue_id != null"
   - rename:
       field: checkpoint.usercheck_incident_uid
       target_field: destination.user.id

packages/checkpoint/data_stream/firewall/fields/ecs.yml

@@ -54,6 +54,22 @@
   name: dns.type
 - external: ecs
   name: ecs.version
+- external: ecs
+  name: email.delivery_timestamp
+- external: ecs
+  name: email.from.address
+- external: ecs
+  name: email.to.address
+- external: ecs
+  name: email.bcc.address
+- external: ecs
+  name: email.cc.address
+- external: ecs
+  name: email.subject
+- external: ecs
+  name: email.message_id
+- external: ecs
+  name: email.local_id
 - external: ecs
   name: error.message
 - external: ecs

packages/checkpoint/data_stream/firewall/sample_event.json

@@ -16,7 +16,7 @@
         "type": "logs"
     },
     "ecs": {
-        "version": "8.0.0"
+        "version": "8.2.0"
     },
     "elastic_agent": {
         "id": "ba9ee39d-37f1-433a-8800-9d424cb9dd11",

packages/checkpoint/docs/README.md

@@ -38,7 +38,7 @@ An example event for `firewall` looks as following:
         "type": "logs"
     },
     "ecs": {
-        "version": "8.0.0"
+        "version": "8.2.0"
     },
     "elastic_agent": {
         "id": "ba9ee39d-37f1-433a-8800-9d424cb9dd11",
@@ -543,6 +543,15 @@ An example event for `firewall` looks as following:
 | dns.question.type | The type of record being queried. | keyword |
 | dns.type | The type of DNS event captured, query or answer. If your source of DNS events only gives you DNS queries, you should only create dns events of type `dns.type:query`. If your source of DNS events gives you answers as well, you should create one event per query (optionally as soon as the query is seen). And a second event containing all query details as well as an array of answers. | keyword |
 | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword |
+| email.bcc.address | The email address of BCC recipient | keyword |
+| email.cc.address | The email address of CC recipient | keyword |
+| email.delivery_timestamp | The date and time when the email message was received by the service or client. | date |
+| email.from.address | The email address of the sender, typically from the RFC 5322 `From:` header field. | keyword |
+| email.local_id | Unique identifier given to the email by the source that created the event. Identifier is not persistent across hops. | keyword |
+| email.message_id | Identifier from the RFC 5322 `Message-ID:` email header that refers to a particular email message. | wildcard |
+| email.subject | A brief summary of the topic of the message. | keyword |
+| email.subject.text | Multi-field of `email.subject`. | match_only_text |
+| email.to.address | The email address of recipient | keyword |
 | error.message | Error message. | match_only_text |
 | event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword |
 | event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword |
@@ -583,7 +592,7 @@ An example event for `firewall` looks as following:
 | host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword |
 | host.os.kernel | Operating system kernel version as a raw string. | keyword |
 | host.os.name | Operating system name, without the version. | keyword |
-| host.os.name.text | Multi-field of `host.os.name`. | match_only_text |
+| host.os.name.text | Multi-field of `host.os.name`. | text |
 | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword |
 | host.os.version | Operating system version as a raw string. | keyword |
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |

packages/checkpoint/manifest.yml

@@ -1,6 +1,6 @@
 name: checkpoint
 title: Check Point
-version: 1.3.6
+version: 1.4.0
 release: ga
 description: Collect logs from Check Point with Elastic Agent.
 type: integration