fix: JSON truncation caused by escaped zero byte (#1594)

Signed-off-by: Tao Yu <yyuuttaaoo@gmail.com>

core/processor/ProcessorParseJsonNative.cpp

@@ -173,7 +173,7 @@ bool ProcessorParseJsonNative::JsonLogLineParser(LogEvent& sourceEvent,
 
 std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Value& value) {
     if (value.IsString())
-        return value.GetString();
+        return std::string(value.GetString(), value.GetStringLength());
     else if (value.IsBool())
         return ToString(value.GetBool());
     else if (value.IsInt())
@@ -193,7 +193,7 @@ std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Va
         rapidjson::StringBuffer buffer;
         rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
         value.Accept(writer);
-        return buffer.GetString();
+        return std::string(buffer.GetString(), buffer.GetLength());
     }
 }
 

core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp

@@ -30,6 +30,7 @@ class ProcessorParseJsonNativeUnittest : public ::testing::Test {
 
     void TestInit();
     void TestProcessJson();
+    void TestProcessJsonEscapedNullByte();
     void TestAddLog();
     void TestProcessEventKeepUnmatch();
     void TestProcessEventDiscardUnmatch();
@@ -46,6 +47,8 @@ UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestAddLog);
 
 UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJson);
 
+UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJsonEscapedNullByte);
+
 UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessEventKeepUnmatch);
 
 UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessEventDiscardUnmatch);
@@ -258,6 +261,63 @@ void ProcessorParseJsonNativeUnittest::TestAddLog() {
                             processor.GetContext().GetProcessProfile().logGroupSize);
 }
 
+void ProcessorParseJsonNativeUnittest::TestProcessJsonEscapedNullByte() {
+    // make config
+    Json::Value config;
+    config["SourceKey"] = "content";
+    config["KeepingSourceWhenParseFail"] = true;
+    config["KeepingSourceWhenParseSucceed"] = false;
+    config["CopingRawLog"] = false;
+    config["RenamedSourceKey"] = "rawLog";
+
+    // make events
+    auto sourceBuffer = std::make_shared<SourceBuffer>();
+    PipelineEventGroup eventGroup(sourceBuffer);
+    std::string inJson = R"({
+        "events" :
+        [
+            {
+                "contents" :
+                {
+                    "content" : "{\"level\":\"ERROR\",\"time\":\"2024-07-04T06:59:23.078Z\",\"msg\":\"expect { or n, but found \\u0000, error found in #0 byte of ...\"}"
+                },
+                "timestampNanosecond" : 0,
+                "timestamp" : 12345678901,
+                "type" : 1
+            }
+        ]
+    })";
+    eventGroup.FromJsonString(inJson);
+    // run function
+    ProcessorParseJsonNative& processor = *(new ProcessorParseJsonNative);
+    std::string pluginId = "testID";
+    ProcessorInstance processorInstance(&processor, pluginId);
+    APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
+    std::vector<PipelineEventGroup> eventGroupList;
+    eventGroupList.emplace_back(std::move(eventGroup));
+    processorInstance.Process(eventGroupList);
+    // judge result
+    std::string expectJson = R"({
+        "events" :
+        [
+            {
+                "contents" :
+                {
+                    "level": "ERROR",
+	                "msg": "expect { or n, but found \u0000, error found in #0 byte of ...",
+	                "time": "2024-07-04T06:59:23.078Z"
+                },
+                "timestamp" : 12345678901,
+                "timestampNanosecond" : 0,
+                "type" : 1
+            }
+        ]
+    })";
+    std::string outJson = eventGroupList[0].ToJsonString();
+    APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
+    APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), uint64_t(0));
+}
+
 void ProcessorParseJsonNativeUnittest::TestProcessJson() {
     // make config
     Json::Value config;