Update TLS parsing for multiple integrations

packages/carbonblack_edr/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.0"
+  changes:
+    - description: Add JA3/JA3S parsing
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3440
 - version: "1.2.0"
   changes:
     - description: Update to ECS 8.2

packages/carbonblack_edr/data_stream/log/_dev/test/pipeline/test-events.json-expected.json

@@ -885,6 +885,9 @@
                     "start"
                 ]
             },
+            "network": {
+                "iana_number": "17"
+            },
             "observer": {
                 "name": "CB_SERVER_HOSTNAME",
                 "product": "Carbon Black EDR",
@@ -1092,6 +1095,9 @@
                     "start"
                 ]
             },
+            "network": {
+                "iana_number": "6"
+            },
             "observer": {
                 "name": "CB_SERVER_HOSTNAME",
                 "product": "Carbon Black EDR",
@@ -1391,6 +1397,9 @@
                     "start"
                 ]
             },
+            "network": {
+                "iana_number": "6"
+            },
             "observer": {
                 "name": "CB_SERVER_HOSTNAME",
                 "product": "Carbon Black EDR",
@@ -1735,16 +1744,33 @@
                     "start"
                 ]
             },
+            "network": {
+                "iana_number": "6"
+            },
             "observer": {
                 "name": "CB_SERVER_HOSTNAME",
                 "product": "Carbon Black EDR",
                 "type": "edr",
                 "vendor": "VMWare"
             },
+            "related": {
+                "hash": [
+                    "eb1d94daa7e0344597e756a1fb6e7054",
+                    "598872011444709307b861ae817a4b60"
+                ]
+            },
             "tags": [
                 "carbonblack_edr-log",
                 "forwarded"
-            ]
+            ],
+            "tls": {
+                "client": {
+                    "ja3": "598872011444709307b861ae817a4b60"
+                },
+                "server": {
+                    "ja3s": "eb1d94daa7e0344597e756a1fb6e7054"
+                }
+            }
         },
         {
             "@timestamp": "2014-09-09T19:00:21.380Z",
@@ -2110,6 +2136,9 @@
                     "start"
                 ]
             },
+            "network": {
+                "iana_number": "17"
+            },
             "observer": {
                 "name": "CB_SERVER_HOSTNAME",
                 "product": "Carbon Black EDR",
@@ -2535,6 +2564,9 @@
                     "start"
                 ]
             },
+            "network": {
+                "iana_number": "6"
+            },
             "observer": {
                 "name": "CB_SERVER_HOSTNAME",
                 "product": "Carbon Black EDR",
@@ -2906,6 +2938,9 @@
                     "start"
                 ]
             },
+            "network": {
+                "iana_number": "17"
+            },
             "observer": {
                 "name": "CB_SERVER_HOSTNAME",
                 "product": "Carbon Black EDR",
@@ -3337,6 +3372,9 @@
                     "start"
                 ]
             },
+            "network": {
+                "iana_number": "6"
+            },
             "observer": {
                 "name": "CB_SERVER_HOSTNAME",
                 "product": "Carbon Black EDR",
@@ -3817,6 +3855,9 @@
                     "start"
                 ]
             },
+            "network": {
+                "iana_number": "17"
+            },
             "observer": {
                 "name": "CB_SERVER_HOSTNAME",
                 "product": "Carbon Black EDR",
@@ -4320,6 +4361,9 @@
                     "start"
                 ]
             },
+            "network": {
+                "iana_number": "17"
+            },
             "observer": {
                 "name": "CB_SERVER_HOSTNAME",
                 "product": "Carbon Black EDR",

packages/carbonblack_edr/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -795,6 +795,20 @@ processors:
     target_field: network.transport
     ignore_missing: true
 
+- convert:
+    field: json.protocol
+    target_field: network.iana_number
+    type: string
+    ignore_missing: true
+- set:
+    field: tls.client.ja3
+    copy_from: json.ja3
+    ignore_empty_value: true
+- set:
+    field: tls.server.ja3s
+    copy_from: json.ja3s
+    ignore_empty_value: true
+
 #
 # Related fields
 #
@@ -817,6 +831,15 @@ processors:
     allow_duplicates: false
     if: 'ctx.process?.parent?.hash?.md5 != null'
 
+- append:
+    field: related.hash
+    value: "{{tls.server.ja3s}}"
+    if: "ctx?.tls?.server?.ja3s != null"
+- append:
+    field: related.hash
+    value: "{{tls.client.ja3}}"
+    if: "ctx?.tls?.client?.ja3 != null"
+    allow_duplicates: false
 #
 # Remove unneeded fields
 #

packages/carbonblack_edr/data_stream/log/fields/ecs.yml

@@ -52,6 +52,8 @@
   external: ecs
 - name: network.transport
   external: ecs
+- name: network.iana_number
+  external: ecs
 - name: observer.name
   external: ecs
 - name: observer.product
@@ -106,3 +108,7 @@
   external: ecs
 - name: threat.indicator.port
   external: ecs
+- name: tls.client.ja3
+  external: ecs
+- name: tls.server.ja3s
+  external: ecs

packages/carbonblack_edr/docs/README.md

@@ -312,6 +312,7 @@ An example event for `log` looks as following:
 | log.offset | Offset of the entry in the log file. | long |
 | log.source.address | Source address from which the log event was read / sent from. | keyword |
 | network.direction | Direction of the network traffic. Recommended values are:   \* ingress   \* egress   \* inbound   \* outbound   \* internal   \* external   \* unknown  When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword |
+| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword |
 | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword |
 | observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword |
 | observer.product | The product name of the observer. | keyword |
@@ -344,5 +345,7 @@ An example event for `log` looks as following:
 | threat.indicator.port | Identifies a threat indicator as a port number (irrespective of direction). | long |
 | threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. Recommended values:   \* autonomous-system   \* artifact   \* directory   \* domain-name   \* email-addr   \* file   \* ipv4-addr   \* ipv6-addr   \* mac-addr   \* mutex   \* port   \* process   \* software   \* url   \* user-account   \* windows-registry-key   \* x509-certificate | keyword |
 | threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword |
+| tls.client.ja3 | A hash that identifies clients based on how they perform an SSL/TLS handshake. | keyword |
+| tls.server.ja3s | A hash that identifies servers based on how they perform an SSL/TLS handshake. | keyword |
 
 

packages/carbonblack_edr/manifest.yml

@@ -1,6 +1,6 @@
 name: carbonblack_edr
 title: VMware Carbon Black EDR
-version: 1.2.0
+version: 1.3.0
 release: ga
 description: Collect logs from VMware Carbon Black EDR with Elastic Agent.
 type: integration

packages/fireeye/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.4.0"
+  changes:
+    - description: Add JA3/JA3S to `related.hash`
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3440
 - version: "1.3.1"
   changes:
     - description: Move invalid field value in sample event file

packages/fireeye/data_stream/nx/_dev/test/pipeline/test-nx.log-expected.json

@@ -399,6 +399,10 @@
                 "vendor": "Fireeye"
             },
             "related": {
+                "hash": [
+                    "9873b112313d7c4e5e8ef6207e6c6f0d",
+                    "21536525fbf9e289f79e0f98af64bb59"
+                ],
                 "ip": [
                     "192.168.1.99",
                     "67.43.156.13"

packages/fireeye/data_stream/nx/elasticsearch/ingest_pipeline/default.yml

@@ -165,6 +165,15 @@ processors:
       value: "{{destination.ip}}"
       allow_duplicates: false
       if: ctx.destination?.ip != null
+  - append:
+      field: related.hash
+      value: "{{tls.server.ja3s}}"
+      if: "ctx?.tls?.server?.ja3s != null"
+  - append:
+      field: related.hash
+      value: "{{tls.client.ja3}}"
+      if: "ctx?.tls?.client?.ja3 != null"
+      allow_duplicates: false
   - remove:
       field:
         - rawmsg

packages/fireeye/data_stream/nx/fields/ecs.yml

@@ -30,6 +30,8 @@
   name: log.file.path
 - external: ecs
   name: related.ip
+- external: ecs
+  name: related.hash
 - external: ecs
   name: source.bytes
 - external: ecs

packages/fireeye/docs/README.md

@@ -114,6 +114,7 @@ The `nx` integration ingests network security logs from FireEye NX through TCP/U
 | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword |
 | observer.product | The product name of the observer. | keyword |
 | observer.vendor | Vendor name of the observer. | keyword |
+| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword |
 | related.ip | All of the IPs seen on your event. | ip |
 | source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword |
 | source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long |

packages/fireeye/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: fireeye
 title: "Fireeye"
-version: 1.3.1
+version: 1.4.0
 license: basic
 description: "This Elastic integration collects Fireeye NX logs."
 type: integration

packages/network_traffic/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.3.0"
+  changes:
+    - description: Add JA3/JA3S to `related.hash`
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3440
 - version: "1.2.0"
   changes:
     - description: Add option to monitor processes.

packages/network_traffic/data_stream/tls/elasticsearch/ingest_pipeline/default.yml

@@ -64,6 +64,16 @@ processors:
             }
         }
 
+- append:
+    field: related.hash
+    value: "{{tls.server.ja3s}}"
+    if: "ctx?.tls?.server?.ja3s != null"
+- append:
+    field: related.hash
+    value: "{{tls.client.ja3}}"
+    if: "ctx?.tls?.client?.ja3 != null"
+    allow_duplicates: false
+
 on_failure:
 - set:
     field: error.message

packages/network_traffic/data_stream/tls/fields/ecs.yml

@@ -44,6 +44,8 @@
   name: network.type
 - external: ecs
   name: related.ip
+- external: ecs
+  name: related.hash
 - external: ecs
   name: server.bytes
 - external: ecs

packages/network_traffic/docs/README.md

@@ -4279,6 +4279,7 @@ Fields published for TLS packets.
 | params | The request parameters. For HTTP, these are the POST or GET parameters. For Thrift-RPC, these are the parameters from the request. | text |
 | path | The path the transaction refers to. For HTTP, this is the URL. For SQL databases, this is the table name. For key-value stores, this is the key. | keyword |
 | query | The query in a human readable format. For HTTP, it will typically be something like `GET /users/_search?name=test`. For MySQL, it is something like `SELECT id from users where name=test`. | keyword |
+| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword |
 | related.ip | All of the IPs seen on your event. | ip |
 | request | For text protocols, this is the request as seen on the wire (application layer only). For binary protocols this is our representation of the request. | text |
 | resource | The logical resource that this transaction refers to. For HTTP, this is the URL path up to the last slash (/). For example, if the URL is `/users/1`, the resource is `/users`. For databases, the resource is typically the table name. The field is not filled for all transaction types. | keyword |

packages/network_traffic/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: network_traffic
 title: Network Packet Capture
-version: 1.2.0
+version: 1.3.0
 license: basic
 description: Capture and analyze network traffic from a host with Elastic Agent.
 type: integration

packages/suricata/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.1.0"
+  changes:
+    - description: Add JA3/JA3S to `related.hash`
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3440
 - version: "2.0.0"
   changes:
     - description: Migrate map visualisation from tile_map to map object

packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-alerts.log-expected.json

@@ -2495,7 +2495,9 @@
             },
             "related": {
                 "hash": [
-                    "00112233445566778899AABBCCDDEEFF00112233"
+                    "00112233445566778899AABBCCDDEEFF00112233",
+                    "0993626a07ad09e1ce91293be7aa5721",
+                    "d92325c876e7279f4eb8c62415e3a6b7"
                 ],
                 "ip": [
                     "10.126.2.140",
@@ -2608,7 +2610,9 @@
             },
             "related": {
                 "hash": [
-                    "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC"
+                    "363FEE2A1CFADEADBEEF4299CFA9B09101EBA9CC",
+                    "391231ba5675e42807b9e1f457b2614e",
+                    "3f1ea03f5822e8021b60cc3e4b233181"
                 ],
                 "ip": [
                     "10.137.3.54",

packages/suricata/data_stream/eve/_dev/test/pipeline/test-eve-small.log-expected.json

@@ -868,7 +868,8 @@
             },
             "related": {
                 "hash": [
-                    "183C114546E926C787640FED47861B31BF0F8425"
+                    "183C114546E926C787640FED47861B31BF0F8425",
+                    "adc06261ef82c2e4688b3cf08c1b2f24"
                 ],
                 "ip": [
                     "192.168.50.1"
@@ -1050,6 +1051,9 @@
                 "transport": "tcp"
             },
             "related": {
+                "hash": [
+                    "44d502d471cfdb99c59bdfb0f220e5a8"
+                ],
                 "ip": [
                     "192.168.50.1"
                 ]

packages/suricata/data_stream/eve/elasticsearch/ingest_pipeline/default.yml

@@ -631,6 +631,15 @@ processors:
       field: related.ip
       value: '{{{destination.ip}}}'
       allow_duplicates: false
+  - append:
+      field: related.hash
+      value: "{{tls.server.ja3s}}"
+      if: "ctx?.tls?.server?.ja3s != null"
+  - append:
+      field: related.hash
+      value: "{{tls.client.ja3}}"
+      if: "ctx?.tls?.client?.ja3 != null"
+      allow_duplicates: false
   - remove:
       field: suricata.eve.alert.metadata
       if: "ctx.suricata?.eve?.alert?.metadata == null || ctx.suricata?.eve?.alert?.metadata.isEmpty()"