fireeye: move invalid field values

packages/fireeye/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.2.4"
+  changes:
+    - description: Move invalid field values
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/3099
 - version: "1.2.3"
   changes:
     - description: Fix typo in config template for ignoring host enrichment

packages/fireeye/data_stream/nx/_dev/test/pipeline/test-common-config.yml

@@ -1,5 +1,3 @@
-dynamic_fields:
-  event.ingested: ".*"
 fields:
   tags:
     - preserve_original_event

packages/fireeye/data_stream/nx/_dev/test/pipeline/test-nx.log-expected.json

@@ -13,9 +13,13 @@
                 "version": "8.0.0"
             },
             "event": {
-                "ingested": "2022-03-26T18:20:15.891947350Z",
+                "category": [
+                    "network"
+                ],
                 "original": "{\"rawmsg\":\"{\\\"timestamp\\\":\\\"2020-09-22T08:34:44.991339+0000\\\",\\\"flow_id\\\":721570461162990,\\\"event_type\\\":\\\"flow\\\",\\\"src_ip\\\":\\\"fe80:0000:0000:0000:feec:daff:fe31:b706\\\",\\\"src_port\\\":45944,\\\"dest_ip\\\":\\\"ff02:0000:0000:0000:0000:0000:0000:0001\\\",\\\"dest_port\\\":10001,\\\"proto\\\":\\\"UDP\\\",\\\"proto_number\\\":17,\\\"ip_tc\\\":0,\\\"app_proto\\\":\\\"failed\\\",\\\"flow\\\":{\\\"pkts_toserver\\\":8,\\\"pkts_toclient\\\":0,\\\"bytes_toserver\\\":1680,\\\"bytes_toclient\\\":0,\\\"start\\\":\\\"2020-09-22T08:34:12.761326+0000\\\",\\\"end\\\":\\\"2020-09-22T08:34:12.761348+0000\\\",\\\"age\\\":0,\\\"state\\\":\\\"new\\\",\\\"reason\\\":\\\"timeout\\\",\\\"alerted\\\":false}}\\n\",\"meta_sip4\":\"192.168.1.99\",\"meta_oml\":520,\"deviceid\":\"860665216674\",\"meta_cbname\":\"fireeye-7e0de1\"}",
-                "type": "flow"
+                "type": [
+                    "info"
+                ]
             },
             "fireeye": {
                 "nx": {
@@ -82,9 +86,13 @@
                 "version": "8.0.0"
             },
             "event": {
-                "ingested": "2022-03-26T18:20:15.891951511Z",
+                "category": [
+                    "network"
+                ],
                 "original": "{\"rawmsg\":\"{\\\"timestamp\\\":\\\"2020-09-22T08:34:44.993228+0000\\\",\\\"flow_id\\\":175370876476591,\\\"event_type\\\":\\\"flow\\\",\\\"src_ip\\\":\\\"192.168.1.15\\\",\\\"src_port\\\":39808,\\\"dest_ip\\\":\\\"67.43.156.14\\\",\\\"dest_port\\\":123,\\\"proto\\\":\\\"UDP\\\",\\\"proto_number\\\":17,\\\"ip_tos\\\":0,\\\"app_proto\\\":\\\"failed\\\",\\\"flow\\\":{\\\"pkts_toserver\\\":1,\\\"pkts_toclient\\\":1,\\\"bytes_toserver\\\":90,\\\"bytes_toclient\\\":90,\\\"start\\\":\\\"2020-09-22T08:33:15.122031+0000\\\",\\\"end\\\":\\\"2020-09-22T08:33:15.193693+0000\\\",\\\"age\\\":0,\\\"state\\\":\\\"established\\\",\\\"reason\\\":\\\"shutdown\\\",\\\"alerted\\\":false}}\\n\",\"meta_sip4\":\"192.168.1.99\",\"meta_oml\":475,\"deviceid\":\"860665216674\",\"meta_cbname\":\"fireeye-7e0de1\"}",
-                "type": "flow"
+                "type": [
+                    "info"
+                ]
             },
             "fireeye": {
                 "nx": {
@@ -139,9 +147,13 @@
                 "version": "8.0.0"
             },
             "event": {
-                "ingested": "2022-03-26T18:20:15.891953408Z",
+                "category": [
+                    "network"
+                ],
                 "original": "{\"rawmsg\":\"{\\\"timestamp\\\":\\\"2020-09-22T08:34:44.993227+0000\\\",\\\"flow_id\\\":1285126005631046,\\\"event_type\\\":\\\"flow\\\",\\\"src_ip\\\":\\\"fe80:0000:0000:0000:feec:daff:fe31:b706\\\",\\\"src_port\\\":44535,\\\"dest_ip\\\":\\\"ff02:0000:0000:0000:0000:0000:0000:0001\\\",\\\"dest_port\\\":10001,\\\"proto\\\":\\\"UDP\\\",\\\"proto_number\\\":17,\\\"ip_tc\\\":0,\\\"app_proto\\\":\\\"failed\\\",\\\"flow\\\":{\\\"pkts_toserver\\\":8,\\\"pkts_toclient\\\":0,\\\"bytes_toserver\\\":1680,\\\"bytes_toclient\\\":0,\\\"start\\\":\\\"2020-09-22T08:34:22.763974+0000\\\",\\\"end\\\":\\\"2020-09-22T08:34:22.764073+0000\\\",\\\"age\\\":0,\\\"state\\\":\\\"new\\\",\\\"reason\\\":\\\"shutdown\\\",\\\"alerted\\\":false}}\\n\",\"meta_sip4\":\"192.168.1.99\",\"meta_oml\":522,\"deviceid\":\"860665216674\",\"meta_cbname\":\"fireeye-7e0de1\"}",
-                "type": "flow"
+                "type": [
+                    "info"
+                ]
             },
             "fireeye": {
                 "nx": {
@@ -208,9 +220,13 @@
                 "version": "8.0.0"
             },
             "event": {
-                "ingested": "2022-03-26T18:20:15.891955258Z",
+                "category": [
+                    "network"
+                ],
                 "original": "{\"rawmsg\":\"{\\\"timestamp\\\":\\\"2020-09-22T08:34:44.993286+0000\\\",\\\"flow_id\\\":222460015300681,\\\"event_type\\\":\\\"flow\\\",\\\"src_ip\\\":\\\"192.168.1.150\\\",\\\"src_port\\\":51082,\\\"dest_ip\\\":\\\"67.43.156.15\\\",\\\"dest_port\\\":5938,\\\"proto\\\":\\\"TCP\\\",\\\"proto_number\\\":6,\\\"ip_tos\\\":0,\\\"app_proto\\\":\\\"failed\\\",\\\"flow\\\":{\\\"pkts_toserver\\\":799,\\\"pkts_toclient\\\":544,\\\"bytes_toserver\\\":69825,\\\"bytes_toclient\\\":59808,\\\"start\\\":\\\"2020-09-22T04:48:48.282697+0000\\\",\\\"end\\\":\\\"2020-09-22T08:34:36.067255+0000\\\",\\\"age\\\":13548,\\\"state\\\":\\\"established\\\",\\\"reason\\\":\\\"shutdown\\\",\\\"alerted\\\":false},\\\"tcp\\\":{\\\"tcp_flags\\\":\\\"1a\\\",\\\"tcp_flags_ts\\\":\\\"1a\\\",\\\"tcp_flags_tc\\\":\\\"1a\\\",\\\"syn\\\":true,\\\"psh\\\":true,\\\"ack\\\":true,\\\"state\\\":\\\"established\\\"}}\\n\",\"meta_sip4\":\"192.168.1.99\",\"meta_oml\":611,\"deviceid\":\"860665216674\",\"meta_cbname\":\"fireeye-7e0de1\"}",
-                "type": "flow"
+                "type": [
+                    "info"
+                ]
             },
             "fireeye": {
                 "nx": {
@@ -286,9 +302,13 @@
                 "version": "8.0.0"
             },
             "event": {
-                "ingested": "2022-03-26T18:20:15.891957082Z",
+                "category": [
+                    "network"
+                ],
                 "original": "{\"rawmsg\":\"{\\\"timestamp\\\":\\\"2020-09-22T08:34:44.993501+0000\\\",\\\"flow_id\\\":1463569002949603,\\\"event_type\\\":\\\"flow\\\",\\\"src_ip\\\":\\\"192.168.1.15\\\",\\\"src_port\\\":52147,\\\"dest_ip\\\":\\\"67.43.156.14\\\",\\\"dest_port\\\":123,\\\"proto\\\":\\\"UDP\\\",\\\"proto_number\\\":17,\\\"ip_tos\\\":0,\\\"app_proto\\\":\\\"failed\\\",\\\"flow\\\":{\\\"pkts_toserver\\\":1,\\\"pkts_toclient\\\":1,\\\"bytes_toserver\\\":90,\\\"bytes_toclient\\\":90,\\\"start\\\":\\\"2020-09-22T08:32:06.355299+0000\\\",\\\"end\\\":\\\"2020-09-22T08:32:06.439495+0000\\\",\\\"age\\\":0,\\\"state\\\":\\\"established\\\",\\\"reason\\\":\\\"shutdown\\\",\\\"alerted\\\":false}}\\n\",\"meta_sip4\":\"192.168.1.99\",\"meta_oml\":476,\"deviceid\":\"860665216674\",\"meta_cbname\":\"fireeye-7e0de1\"}",
-                "type": "flow"
+                "type": [
+                    "info"
+                ]
             },
             "fireeye": {
                 "nx": {
@@ -353,9 +373,13 @@
                 "version": "8.0.0"
             },
             "event": {
-                "ingested": "2022-03-26T18:20:15.891958935Z",
+                "category": [
+                    "network"
+                ],
                 "original": "{\"rawmsg\":\"{\\\"timestamp\\\":\\\"2020-09-23T05:02:01.175635+0000\\\",\\\"flow_id\\\":1136872856843530,\\\"iface\\\":\\\"pether3\\\",\\\"event_type\\\":\\\"tls\\\",\\\"src_ip\\\":\\\"192.168.1.99\\\",\\\"src_port\\\":53918,\\\"dest_ip\\\":\\\"67.43.156.13\\\",\\\"dest_port\\\":443,\\\"proto\\\":\\\"TCP\\\",\\\"tls\\\":{\\\"subject\\\":\\\"C=US, ST=CA, L=San Francisco, O=Cloudflare, Inc., CN=fireeye.com\\\",\\\"issuerdn\\\":\\\"C=US, O=Cloudflare, Inc., CN=Cloudflare Inc ECC CA-3\\\",\\\"ja3\\\":{\\\"hash\\\":\\\"21536525fbf9e289f79e0f98af64bb59\\\",\\\"string\\\":\\\"771,49199-49195-49200-49196-158-159-49191-49187-49192-49188-103-107-49171-49161-49172-49162-51-57-156-157-60-61-47-53-255,0-11-10-13-15-13172,25-24-23,0-1-2\\\"},\\\"ja3s\\\":{\\\"hash\\\":\\\"9873b112313d7c4e5e8ef6207e6c6f0d\\\",\\\"string\\\":\\\"771,49195,0-65281-11-13172\\\"},\\\"fingerprint\\\":\\\"2a:6a:46:d2:05:4d:7b:22:1b:68:02:f2:ee:f0:09:c6:ff:15:e9:58\\\",\\\"sni\\\":\\\"cloud.fireeye.com\\\",\\\"version\\\":\\\"TLS 1.2\\\",\\\"notbefore\\\":\\\"2020-07-01T00:00:00.000000+0000\\\",\\\"notafter\\\":\\\"2021-07-01T12:00:00.000000+0000\\\",\\\"client_ciphersuites\\\":[49199,49195,49200,49196,158,159,49191,49187,49192,49188,103,107,49171,49161,49172,49162,51,57,156,157,60,61,47,53,255],\\\"client_tls_exts\\\":[0,11,10,13,15,13172],\\\"server_ciphersuite\\\":49195,\\\"server_tls_exts\\\":[0,65281,11,13172],\\\"pubkeylength\\\":65}}\\n\",\"meta_sip4\":\"192.168.1.99\",\"meta_oml\":1146,\"deviceid\":\"860665216674\",\"meta_cbname\":\"fireeye-7e0de1\"}",
-                "type": "tls"
+                "type": [
+                    "info"
+                ]
             },
             "fireeye": {
                 "nx": {
@@ -460,9 +484,14 @@
                 "version": "8.0.0"
             },
             "event": {
-                "ingested": "2022-03-26T18:20:15.891960832Z",
+                "category": [
+                    "file",
+                    "network"
+                ],
                 "original": "{\"rawmsg\":\"{\\\"timestamp\\\":\\\"2020-09-23T05:02:19.906154+0000\\\",\\\"flow_id\\\":1444203537876422,\\\"iface\\\":\\\"pether3\\\",\\\"event_type\\\":\\\"fileinfo\\\",\\\"src_ip\\\":\\\"192.168.1.222\\\",\\\"src_port\\\":47220,\\\"dest_ip\\\":\\\"192.168.100.31\\\",\\\"dest_port\\\":5601,\\\"proto\\\":\\\"TCP\\\",\\\"http\\\":{\\\"hostname\\\":\\\"192.168.100.31\\\",\\\"url\\\":\\\"\\\\/internal\\\\/search\\\\/es\\\",\\\"http_user_agent\\\":\\\"Mozilla\\\\/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\\\\/537.36 (KHTML, like Gecko) Chrome\\\\/85.0.4183.102 Safari\\\\/537.36\\\",\\\"http_refer\\\":\\\"http:\\\\/\\\\/192.168.100.31:5601\\\\/app\\\\/kibana\\\",\\\"http_method\\\":\\\"POST\\\",\\\"protocol\\\":\\\"HTTP\\\\/1.1\\\",\\\"length\\\":0},\\\"app_proto\\\":\\\"http\\\",\\\"fileinfo\\\":{\\\"filename\\\":\\\"\\\\/internal\\\\/search\\\\/es\\\",\\\"magic\\\":\\\"ASCII text, with very long lines, with no line terminators\\\",\\\"state\\\":\\\"CLOSED\\\",\\\"md5\\\":\\\"548d03d3e11c009da833e6e59c4adfee\\\",\\\"stored\\\":false,\\\"size\\\":6394,\\\"tx_id\\\":0}}\\n\",\"meta_sip4\":\"192.168.1.99\",\"meta_oml\":769,\"deviceid\":\"860665216674\",\"meta_cbname\":\"fireeye-7e0de1\"}",
-                "type": "fileinfo"
+                "type": [
+                    "info"
+                ]
             },
             "fireeye": {
                 "nx": {
@@ -563,9 +592,13 @@
                 "version": "8.0.0"
             },
             "event": {
-                "ingested": "2022-03-26T18:20:15.891962649Z",
+                "category": [
+                    "network"
+                ],
                 "original": "{\"rawmsg\":\"{\\\"timestamp\\\":\\\"2020-09-23T05:02:41.077232+0000\\\",\\\"flow_id\\\":206535698492848,\\\"iface\\\":\\\"pether3\\\",\\\"event_type\\\":\\\"dns\\\",\\\"src_ip\\\":\\\"192.168.1.176\\\",\\\"src_port\\\":60269,\\\"dest_ip\\\":\\\"67.43.156.15\\\",\\\"dest_port\\\":53,\\\"proto\\\":\\\"UDP\\\",\\\"dns\\\":{\\\"type\\\":\\\"query\\\",\\\"id\\\":28224,\\\"rrname\\\":\\\"time-ios.apple.com\\\",\\\"rrtype\\\":\\\"A\\\",\\\"tx_id\\\":0}}\\n\",\"meta_sip4\":\"192.168.1.99\",\"meta_oml\":289,\"deviceid\":\"860665216674\",\"meta_cbname\":\"fireeye-7e0de1\"}",
-                "type": "dns"
+                "type": [
+                    "info"
+                ]
             },
             "fireeye": {
                 "nx": {

packages/fireeye/data_stream/nx/elasticsearch/ingest_pipeline/default.yml

@@ -1,9 +1,6 @@
 ---
 description: Pipeline for processing FireEye NX logs
 processors:
-  - set:
-      field: event.ingested
-      value: "{{_ingest.timestamp}}"
   - set:
       field: ecs.version
       value: "8.0.0"
@@ -77,6 +74,23 @@ processors:
   - user_agent:
       field: user_agent.original
       ignore_missing: true
+
+  - append:
+      field: event.category
+      value: network
+      if: "['dns', 'flow', 'tls'].contains(ctx?.event?.type)"
+  - append:
+      field: event.category
+      value: [web, network]
+      if: ctx?.event?.type == 'http'
+  - append:
+      field: event.category
+      value: [file, network]
+      if: ctx?.event?.type == 'fileinfo'
+  - set:
+      field: event.type
+      value: [info]
+
   #
   # Normalize protocol names
   #

packages/fireeye/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: fireeye
 title: "Fireeye"
-version: 1.2.3
+version: 1.2.4
 license: basic
 description: "This Elastic integration collects Fireeye NX logs."
 type: integration