fix: refactor multiline split state

core/file_server/MultilineOptions.cpp

@@ -166,6 +166,19 @@ bool MultilineOptions::Init(const Json::Value& config, const PipelineContext& ct
                               ctx.GetRegion());
     }
 
+    // Ignore Warning
+    if (!GetOptionalBoolParam(config, "IgnoringUnmatchWarning", mIgnoringUnmatchWarning, errorMsg)) {
+        PARAM_WARNING_DEFAULT(ctx.GetLogger(),
+                              ctx.GetAlarm(),
+                              errorMsg,
+                              mIgnoringUnmatchWarning,
+                              pluginName,
+                              ctx.GetConfigName(),
+                              ctx.GetProjectName(),
+                              ctx.GetLogstoreName(),
+                              ctx.GetRegion());
+    }
+
     return true;
 }
 
@@ -181,4 +194,19 @@ bool MultilineOptions::ParseRegex(const string& pattern, shared_ptr<boost::regex
     return true;
 }
 
+const std::string&
+UnmatchedContentTreatmentToString(MultilineOptions::UnmatchedContentTreatment unmatchedContentTreatment) {
+    switch (unmatchedContentTreatment) {
+        case MultilineOptions::UnmatchedContentTreatment::DISCARD:
+            static std::string discardStr = "discard";
+            return discardStr;
+        case MultilineOptions::UnmatchedContentTreatment::SINGLE_LINE:
+            static std::string singleLine = "single line";
+            return singleLine;
+        default:
+            static std::string unkonwn = "";
+            return unkonwn;
+    }
+}
+
 } // namespace logtail

core/file_server/MultilineOptions.h

@@ -42,6 +42,7 @@ class MultilineOptions {
     std::string mContinuePattern;
     std::string mEndPattern;
     UnmatchedContentTreatment mUnmatchedContentTreatment = UnmatchedContentTreatment::SINGLE_LINE;
+    bool mIgnoringUnmatchWarning = false;
 
 private:
     bool ParseRegex(const std::string& pattern, std::shared_ptr<boost::regex>& reg);
@@ -52,6 +53,9 @@ class MultilineOptions {
     bool mIsMultiline = false;
 };
 
+const std::string&
+UnmatchedContentTreatmentToString(MultilineOptions::UnmatchedContentTreatment unmatchedContentTreatment);
+
 using MultilineConfig = std::pair<const MultilineOptions*, const PipelineContext*>;
 
 } // namespace logtail

core/monitor/MetricConstants.cpp

@@ -43,6 +43,12 @@ const std::string METRIC_PROC_PARSE_ERROR_TOTAL = "proc_parse_error_total";
 const std::string METRIC_PROC_KEY_COUNT_NOT_MATCH_ERROR_TOTAL = "proc_key_count_not_match_error_total";
 const std::string METRIC_PROC_HISTORY_FAILURE_TOTAL = "proc_history_failure_total";
 
+const std::string METRIC_PROC_SPLIT_MULTILINE_LOG_MATCHED_RECORDS_TOTAL
+    = "proc_split_multiline_log_matched_records_total";
+const std::string METRIC_PROC_SPLIT_MULTILINE_LOG_MATCHED_LINES_TOTAL = "proc_split_multiline_log_matched_lines_total";
+const std::string METRIC_PROC_SPLIT_MULTILINE_LOG_UNMATCHED_LINES_TOTAL
+    = "proc_split_multiline_log_unmatched_lines_total";
+
 // processor filter metrics
 const std::string METRIC_PROC_FILTER_IN_SIZE_BYTES = "proc_filter_in_size_bytes";
 const std::string METRIC_PROC_FILTER_OUT_SIZE_BYTES = "proc_filter_out_size_bytes";
@@ -56,4 +62,4 @@ const std::string PLUGIN_PROCESSOR_PARSE_REGEX_NATIVE = "processor_parse_regex_n
 // processor desensitize metrics
 const std::string METRIC_PROC_DESENSITIZE_RECORDS_TOTAL = "proc_desensitize_records_total";
 
-}
+} // namespace logtail

core/monitor/MetricConstants.h

@@ -42,6 +42,9 @@ extern const std::string METRIC_PROC_PARSE_OUT_SIZE_BYTES;
 extern const std::string METRIC_PROC_PARSE_ERROR_TOTAL;
 extern const std::string METRIC_PROC_KEY_COUNT_NOT_MATCH_ERROR_TOTAL;
 extern const std::string METRIC_PROC_HISTORY_FAILURE_TOTAL;
+extern const std::string METRIC_PROC_SPLIT_MULTILINE_LOG_MATCHED_RECORDS_TOTAL;
+extern const std::string METRIC_PROC_SPLIT_MULTILINE_LOG_MATCHED_LINES_TOTAL;
+extern const std::string METRIC_PROC_SPLIT_MULTILINE_LOG_UNMATCHED_LINES_TOTAL;
 
 // processor filter metrics
 extern const std::string METRIC_PROC_FILTER_IN_SIZE_BYTES;

core/pipeline/Pipeline.cpp

@@ -26,7 +26,7 @@
 #include "plugin/PluginRegistry.h"
 #include "processor/ProcessorParseApsaraNative.h"
 #include "processor/ProcessorSplitLogStringNative.h"
-#include "processor/ProcessorSplitRegexNative.h"
+#include "processor/ProcessorSplitMultilineLogStringNative.h"
 #include "processor/ProcessorTagNative.h"
 #include "processor/daemon/LogProcess.h"
 
@@ -113,7 +113,7 @@ bool Pipeline::Init(Config&& config) {
             detail["SplitChar"] = Json::Value('\0');
             detail["AppendingLogPositionMeta"] = Json::Value(inputFile->mFileReader.mAppendingLogPositionMeta);
         } else if (inputFile->mMultiline.IsMultiline()) {
-            processor = PluginRegistry::GetInstance()->CreateProcessor(ProcessorSplitRegexNative::sName,
+            processor = PluginRegistry::GetInstance()->CreateProcessor(ProcessorSplitMultilineLogStringNative::sName,
                                                                        to_string(++pluginIndex));
             detail["Mode"] = Json::Value("custom");
             detail["StartPattern"] = Json::Value(inputFile->mMultiline.mStartPattern);

core/plugin/PluginRegistry.cpp

@@ -46,7 +46,7 @@
 #include "processor/ProcessorParseRegexNative.h"
 #include "processor/ProcessorParseTimestampNative.h"
 #include "processor/ProcessorSplitLogStringNative.h"
-#include "processor/ProcessorSplitRegexNative.h"
+#include "processor/ProcessorSplitMultilineLogStringNative.h"
 #include "processor/ProcessorTagNative.h"
 #if defined(__linux__) && !defined(__ANDROID__)
 #include "processor/ProcessorSPL.h"
@@ -250,7 +250,7 @@ void PluginRegistry::LoadStaticPlugins() {
 #endif
 
     RegisterProcessorCreator(new StaticProcessorCreator<ProcessorSplitLogStringNative>());
-    RegisterProcessorCreator(new StaticProcessorCreator<ProcessorSplitRegexNative>());
+    RegisterProcessorCreator(new StaticProcessorCreator<ProcessorSplitMultilineLogStringNative>());
     RegisterProcessorCreator(new StaticProcessorCreator<ProcessorParseApsaraNative>());
     RegisterProcessorCreator(new StaticProcessorCreator<ProcessorParseDelimiterNative>());
     RegisterProcessorCreator(new StaticProcessorCreator<ProcessorDesensitizeNative>());