XSUP-42968 Linux Parsing Rule Extension for Additional Timestamp Form…

…ats (#37039)

* add support for ISO 8601 with spaces

* fix Zulu time notation format

* add support for timestamp in embedded json payloads or given as a key-value pair

* added support for offset hh:mm separated by space

* update release notes

Packs/LinuxEventsCollection/ParsingRules/LinuxEventsCollectionParsingRules/LinuxEventsCollectionParsingRules.xif

@@ -1,16 +1,23 @@
 [INGEST:vendor="linux", product="linux", target_dataset="linux_linux_raw", no_hit=keep]
 // Filter applies to log records which contain a full RFC 3339 style timestamp with an explicit timezone offset or Zulu time suffix (UTC)
-filter _raw_log ~= "\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:?\d{2}|Z)"
-| alter tmp_timestamp = arrayindex(regextract(_raw_log, "\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}\S+"), 0)
+filter _raw_log ~= "\d{4}\-\d{2}\-\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:\.\d+)?\s*(?:[+-]\d{2}:?\d{2}|Z)"
+| alter tmp_json_payload = arrayindex(regextract(_raw_log, "({.+})"), 0) // extract inner json payload from _raw_log if such exists
+| alter tmp_timestamp = coalesce( // extract timestamp field from json payload if it exists or the raw text if it does not
+    tmp_json_payload -> time,
+    tmp_json_payload -> Data.Client.CurrentDateTime, 
+    arrayindex(regextract(_raw_log, "(\d{4}\-\d{2}\-\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:\.\d+)?\s*(?:[+-]\d{2}:?\d{2}|Z))"), 0))
 | alter _time = if(
+    tmp_timestamp ~= "\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}(?:\.\d+)?\s[+-]\d{4}", parse_timestamp("%F %H:%M:%E*S %z", tmp_timestamp), // ISO 8601 compatible, with spaces & timezone offset, e.g., 2024-11-04 09:46:36.885 +1100
+    tmp_timestamp ~= "\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}(?:\.\d+)?\s[+-]\d{2}:\d{2}", parse_timestamp("%F %H:%M:%E*S %Ez", tmp_timestamp), // ISO 8601 compatible, with spaces & timezone offset, and offset with colon, e.g., 2024-11-04 09:46:36.885 +11:00
+    tmp_timestamp ~= "\d{4}\-\d{2}\-\d{2}\s\d{2}:\d{2}:\d{2}(?:\.\d+)?\sZ", parse_timestamp("%F %H:%M:%E*S Z", tmp_timestamp), // ISO 8601 compatible, with spaces in zulu time notation, e.g., 2024-11-04 09:46:36.885 Z
     tmp_timestamp ~= "[+-]\d{4}", parse_timestamp("%FT%H:%M:%E*S%z", tmp_timestamp), // timezone offset without a separating colon, e.g., 2024-10-28T14:30:55+0300
     tmp_timestamp ~= "[+-]\d{2}:\d{2}", parse_timestamp("%FT%H:%M:%E*S%Ez", tmp_timestamp), // RFC 3339 compatible timestamp offset, e.g., 2024-10-28T14:30:55+03:00
     parse_timestamp("%FT%H:%M:%E*SZ", tmp_timestamp)) // RFC 3339 compatible timestamp with zulu time notation, e.g., 2024-10-28T14:30:55Z
 | fields - tmp*;
 
 /* Filter applies to RFC 3164 style syslog records (logs timestamp does not contain an explicit timezone nor year), which are ingested via the Broker VM syslog applet 
    Log records which are ingested via the XDR Filebeat collector or excluded since their timestamp assignment is handled in the backend within the XDRC based on the Filebeat @timestamp entity. */
-filter _collector_type != "XDR Collector" and _raw_log ~= "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}" and _raw_log !~= "\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:?\d{2}|Z)"
+filter _collector_type != "XDR Collector" and _raw_log ~= "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}" and _raw_log !~= "\d{4}\-\d{2}\-\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:\.\d+)?\s*(?:[+-]\d{2}:?\d{2}|Z)"
 | alter // construct timestamp basted on log timestamp and current year, assuming timestamp is given in UTC 
     tmp_current_year = format_timestamp("%Y", _insert_time), 
     tmp_rfc_3164_timestamp = arrayindex(regextract(_raw_log, "\w{3}\s+\d{1,2}\s\d{2}:\d{2}:\d{2}"), 0) // RFC 3164 compatible timestamp, e.g., Oct 28 14:30:55

Packs/LinuxEventsCollection/ReleaseNotes/1_0_11.md

@@ -0,0 +1,7 @@
+
+#### Parsing Rules
+
+##### Linux Events Collection Parsing Rule
+
+- Added support for an additional profile of ISO 8601, that uses spaces between the date, time, and timezone offset components, such as `2024-11-04 10:36:12.818 +1100`.
+- Added support for extracting timestamps from key-value pairs embedded in the raw log. 

Packs/LinuxEventsCollection/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Linux Events Collection",
     "description": "Linux is an operating system for servers, desktops, cloud, and IoTs",
     "support": "xsoar",
-    "currentVersion": "1.0.10",
+    "currentVersion": "1.0.11",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",