zeek add support for iso8601 timestamps

- relates elastic/beats#25564

Author: leehinman

packages/zeek/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.8.1"
+  changes:
+    - description: Add support for ISO8601 timestamps
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1118
 - version: "0.8.0"
   changes:
     - description: Update to ECS 1.10.0, adding processor fields and replacing default tags from . to - between words.

packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json

@@ -15,7 +15,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-14T08:33:52.840653500Z",
+                "ingested": "2021-06-17T13:41:22.759516Z",
                 "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
@@ -40,7 +40,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-14T08:33:52.840677100Z",
+                "ingested": "2021-06-17T13:41:22.759526400Z",
                 "original": "{\"ts\":1617062640.941952,\"ts_delta\":900.0005369186401,\"peer\":\"zeek\",\"gaps\":58475,\"acks\":65665,\"percent_lost\":89.05048351481003}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
@@ -65,7 +65,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-14T08:33:52.840684700Z",
+                "ingested": "2021-06-17T13:41:22.759530800Z",
                 "original": "{\"ts\":1617063540.942231,\"ts_delta\":900.0002789497376,\"peer\":\"zeek\",\"gaps\":54754,\"acks\":61818,\"percent_lost\":88.5729075673752}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
@@ -90,7 +90,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-14T08:33:52.840750500Z",
+                "ingested": "2021-06-17T13:41:22.759536200Z",
                 "original": "{\"ts\":1617064440.942597,\"ts_delta\":900.0003659725189,\"peer\":\"zeek\",\"gaps\":51022,\"acks\":57974,\"percent_lost\":88.00841756649533}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
@@ -115,7 +115,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-14T08:33:52.840757600Z",
+                "ingested": "2021-06-17T13:41:22.759543500Z",
                 "original": "{\"ts\":1617065340.942651,\"ts_delta\":900.0000541210175,\"peer\":\"zeek\",\"gaps\":55105,\"acks\":62497,\"percent_lost\":88.17223226714883}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
@@ -148,7 +148,7 @@
                 }
             },
             "event": {
-                "ingested": "2021-06-14T08:33:52.840763500Z",
+                "ingested": "2021-06-17T13:41:22.759548900Z",
                 "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",

packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml

@@ -62,6 +62,7 @@ processors:
       field: zeek.capture_loss.ts
       formats:
         - UNIX
+        - ISO8601
   - set:
       field: event.kind
       value: metric

packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log

@@ -14,4 +14,5 @@
 {"ts":1617062400.703865,"uid":"C3pPjh1YRYcVDiZD3","id.orig_h":"10.156.0.2","id.orig_p":44944,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
 {"ts":1617062400.703851,"uid":"ChUxTmYLG37oO5qUb","id.orig_h":"10.156.0.2","id.orig_p":44942,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
 {"ts":1617062400.704467,"uid":"CpeAOT3B11CTXJgzw2","id.orig_h":"10.156.0.2","id.orig_p":44946,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}
-{"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}}
+{"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}}
+{"ts":"2021-06-09T20:55:13.160328Z","uid":"C2KP1V3alRLoxl4JB9","id.orig_h":"10.0.2.15","id.orig_p":46408,"id.resp_h":"172.217.9.68","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0}