update regex for syslog, add to ignore header

nikhilsinhaparseable · nikhilsinhaparseable · commit fd65c35ced72 · 2025-03-22T05:46:34.000-04:00
diff --git a/resources/formats.json b/resources/formats.json
@@ -384,6 +384,18 @@
         {
           "pattern": "^<(?P<log_pri>\\d+)>(?P<syslog_version>\\d+) (?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{6})?(?:[^ ]+)?) (?P<log_hostname>[^ ]+|-) (?P<log_syslog_tag>(?P<log_procname>[^ ]+|-) (?P<log_pid>[^ ]+|-) (?P<log_msgid>[^ ]+|-)) (?P<log_struct>\\[(?:[^\\]\"]|\"(?:\\.|[^\"])+\")*\\]|-|)\\s+(?P<body>.*)",
           "fields": ["log_pri", "syslog_version", "timestamp", "log_hostname", "log_syslog_tag", "log_procname", "log_pid", "log_msgid", "log_struct", "body"]
+        },
+        {
+          "pattern": "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+\\+\\d{2}:\\d{2})\\s+(?P<hostname>\\S+)\\s+(?P<log_source>)\\[(?P<pid>\\d+)\\]:\\s+\\[(?P<metric_timestamp>\\d+)\\]\\s+cpu\\.local:\\s+\\[\\[(?P<unix_time>\\d+\\.\\d+),\\s+\\{\\}\\],\\s+\\{(?P<cpu_metrics>.*)\\}\\]$",
+          "fields": ["timestamp", "hostname", "log_source", "pid", "metric_timestamp", "unix_time", "cpu_metrics"]
+        },
+        {
+          "pattern": "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+\\+\\d{2}:\\d{2})\\s+(?P<hostname>\\S+)\\s+(?P<log_source>)\\[(?P<pid>\\d+)\\]:\\s+\\[(?P<log_timestamp>\\d{4}/\\d{2}/\\d{2}\\s+\\d{2}:\\d{2}:\\d{2})\\]\\s+\\[(?P<level>info|error|warn)\\]\\s+\\[(?P<component>output:http:http\\.\\d+)\\]\\s+(?P<message>.*)$",
+          "fields": ["timestamp", "hostname", "log_source", "pid", "log_timestamp", "level", "component", "message"]
+        },
+        {
+          "pattern": "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+\\+\\d{2}:\\d{2})\\s+(?P<hostname>\\S+)\\s+(?P<log_source>)\\[(?P<pid>\\d+)\\]:(?P<message>\\S+)$",
+          "fields": ["timestamp", "hostname", "log_source", "pid", "message"]
         }
       ]
     },
diff --git a/src/handlers/http/ingest.rs b/src/handlers/http/ingest.rs
@@ -82,9 +82,6 @@ pub async fn ingest(
     }
 
     let fields = match &log_source {
-        LogSource::OtelLogs | LogSource::OtelMetrics | LogSource::OtelTraces => {
-            return Err(PostError::OtelNotSupported)
-        }
         LogSource::Custom(src) => {
             KNOWN_SCHEMA_LIST.extract_from_inline_log(&mut json, src, extract_log)?
         }
diff --git a/src/handlers/http/modal/utils/ingest_utils.rs b/src/handlers/http/modal/utils/ingest_utils.rs
@@ -36,15 +36,15 @@ use crate::{
             ingest::PostError,
             kinesis::{flatten_kinesis_logs, Message},
         },
-        LOG_SOURCE_KEY, STREAM_NAME_HEADER_KEY,
+        EXTRACT_LOG_KEY, LOG_SOURCE_KEY, STREAM_NAME_HEADER_KEY,
     },
     otel::{logs::flatten_otel_logs, metrics::flatten_otel_metrics, traces::flatten_otel_traces},
     parseable::PARSEABLE,
     storage::StreamType,
     utils::json::{convert_array_to_object, flatten::convert_to_array},
 };
 
-const IGNORE_HEADERS: [&str; 2] = [STREAM_NAME_HEADER_KEY, LOG_SOURCE_KEY];
+const IGNORE_HEADERS: [&str; 3] = [STREAM_NAME_HEADER_KEY, LOG_SOURCE_KEY, EXTRACT_LOG_KEY];
 const MAX_CUSTOM_FIELDS: usize = 10;
 const MAX_FIELD_VALUE_LENGTH: usize = 100;
 

Original file line number	Diff line number	Diff line change
`@@ -384,6 +384,18 @@`
`384`	`384`	`{`
`385`	`385`	`"pattern": "^<(?P<log_pri>\\d+)>(?P<syslog_version>\\d+) (?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{6})?(?:[^ ]+)?) (?P<log_hostname>[^ ]+\|-) (?P<log_syslog_tag>(?P<log_procname>[^ ]+\|-) (?P<log_pid>[^ ]+\|-) (?P<log_msgid>[^ ]+\|-)) (?P<log_struct>\\[(?:[^\\]\"]\|\"(?:\\.\|[^\"])+\")\\]\|-\|)\\s+(?P<body>.)",`
`386`	`386`	`"fields": ["log_pri", "syslog_version", "timestamp", "log_hostname", "log_syslog_tag", "log_procname", "log_pid", "log_msgid", "log_struct", "body"]`
	`387`	`+ },`
	`388`	`+ {`
	`389`	`+ "pattern": "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+\\+\\d{2}:\\d{2})\\s+(?P<hostname>\\S+)\\s+(?P<log_source>)\\[(?P<pid>\\d+)\\]:\\s+\\[(?P<metric_timestamp>\\d+)\\]\\s+cpu\\.local:\\s+\\[\\[(?P<unix_time>\\d+\\.\\d+),\\s+\\{\\}\\],\\s+\\{(?P<cpu_metrics>.*)\\}\\]$",`
	`390`	`+ "fields": ["timestamp", "hostname", "log_source", "pid", "metric_timestamp", "unix_time", "cpu_metrics"]`
	`391`	`+ },`
	`392`	`+ {`
	`393`	`+ "pattern": "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+\\+\\d{2}:\\d{2})\\s+(?P<hostname>\\S+)\\s+(?P<log_source>)\\[(?P<pid>\\d+)\\]:\\s+\\[(?P<log_timestamp>\\d{4}/\\d{2}/\\d{2}\\s+\\d{2}:\\d{2}:\\d{2})\\]\\s+\\[(?P<level>info\|error\|warn)\\]\\s+\\[(?P<component>output:http:http\\.\\d+)\\]\\s+(?P<message>.*)$",`
	`394`	`+ "fields": ["timestamp", "hostname", "log_source", "pid", "log_timestamp", "level", "component", "message"]`
	`395`	`+ },`
	`396`	`+ {`
	`397`	`+ "pattern": "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+\\+\\d{2}:\\d{2})\\s+(?P<hostname>\\S+)\\s+(?P<log_source>)\\[(?P<pid>\\d+)\\]:(?P<message>\\S+)$",`
	`398`	`+ "fields": ["timestamp", "hostname", "log_source", "pid", "message"]`
`387`	`399`	`}`
`388`	`400`	`]`
`389`	`401`	`},`
Original file line number	Diff line number	Diff line change
`@@ -82,9 +82,6 @@ pub async fn ingest(`
`82`	`82`	`}`
`83`	`83`
`84`	`84`	`let fields = match &log_source {`
`85`		`- LogSource::OtelLogs \| LogSource::OtelMetrics \| LogSource::OtelTraces => {`
`86`		`- return Err(PostError::OtelNotSupported)`
`87`		`- }`
`88`	`85`	`LogSource::Custom(src) => {`
`89`	`86`	`KNOWN_SCHEMA_LIST.extract_from_inline_log(&mut json, src, extract_log)?`
`90`	`87`	`}`