From a5a3cd23dfcacff3ea37c8b0c72ff6a2c9858464 Mon Sep 17 00:00:00 2001
From: Tom Yu <yyuuttaaoo@gmail.com>
Date: Tue, 9 Jul 2024 11:52:36 +0800
Subject: [PATCH] fix: JSON truncation caused by escaped zero byte (#1594)

Signed-off-by: Tao Yu <yyuuttaaoo@gmail.com>
---
 core/processor/ProcessorParseJsonNative.cpp   |  4 +-
 .../ProcessorParseJsonNativeUnittest.cpp      | 60 +++++++++++++++++++
 2 files changed, 62 insertions(+), 2 deletions(-)

diff --git a/core/processor/ProcessorParseJsonNative.cpp b/core/processor/ProcessorParseJsonNative.cpp
index 08737b12f9..08f929e581 100644
--- a/core/processor/ProcessorParseJsonNative.cpp
+++ b/core/processor/ProcessorParseJsonNative.cpp
@@ -176,7 +176,7 @@ bool ProcessorParseJsonNative::JsonLogLineParser(LogEvent& sourceEvent,
 
 std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Value& value) {
     if (value.IsString())
-        return value.GetString();
+        return std::string(value.GetString(), value.GetStringLength());
     else if (value.IsBool())
         return ToString(value.GetBool());
     else if (value.IsInt())
@@ -196,7 +196,7 @@ std::string ProcessorParseJsonNative::RapidjsonValueToString(const rapidjson::Va
         rapidjson::StringBuffer buffer;
         rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
         value.Accept(writer);
-        return buffer.GetString();
+        return std::string(buffer.GetString(), buffer.GetLength());
     }
 }
 
diff --git a/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp b/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp
index b229d1dbd4..cafd48060b 100644
--- a/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp
+++ b/core/unittest/processor/ProcessorParseJsonNativeUnittest.cpp
@@ -35,6 +35,7 @@ class ProcessorParseJsonNativeUnittest : public ::testing::Test {
 
     void TestInit();
     void TestProcessJson();
+    void TestProcessJsonEscapedNullByte();
     void TestAddLog();
     void TestProcessEventKeepUnmatch();
     void TestProcessEventDiscardUnmatch();
@@ -51,6 +52,8 @@ UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestAddLog);
 
 UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJson);
 
+UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessJsonEscapedNullByte);
+
 UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessEventKeepUnmatch);
 
 UNIT_TEST_CASE(ProcessorParseJsonNativeUnittest, TestProcessEventDiscardUnmatch);
@@ -268,6 +271,63 @@ void ProcessorParseJsonNativeUnittest::TestAddLog() {
     APSARA_TEST_EQUAL_FATAL(int(strlen(key) + strlen(value) + 5), processor.GetContext().GetProcessProfile().logGroupSize);
 }
 
+void ProcessorParseJsonNativeUnittest::TestProcessJsonEscapedNullByte() {
+    // make config
+    Json::Value config;
+    config["SourceKey"] = "content";
+    config["KeepingSourceWhenParseFail"] = true;
+    config["KeepingSourceWhenParseSucceed"] = false;
+    config["CopingRawLog"] = false;
+    config["RenamedSourceKey"] = "rawLog";
+
+    // make events
+    auto sourceBuffer = std::make_shared<SourceBuffer>();
+    PipelineEventGroup eventGroup(sourceBuffer);
+    std::string inJson = R"({
+        "events" :
+        [
+            {
+                "contents" :
+                {
+                    "content" : "{\"level\":\"ERROR\",\"time\":\"2024-07-04T06:59:23.078Z\",\"msg\":\"expect { or n, but found \\u0000, error found in #0 byte of ...\"}"
+                },
+                "timestampNanosecond" : 0,
+                "timestamp" : 12345678901,
+                "type" : 1
+            }
+        ]
+    })";
+    eventGroup.FromJsonString(inJson);
+    // run function
+    ProcessorParseJsonNative& processor = *(new ProcessorParseJsonNative);
+    std::string pluginId = "testID";
+    ProcessorInstance processorInstance(&processor, pluginId);
+    APSARA_TEST_TRUE_FATAL(processorInstance.Init(config, mContext));
+    std::vector<PipelineEventGroup> eventGroupList;
+    eventGroupList.emplace_back(std::move(eventGroup));
+    processorInstance.Process(eventGroupList);
+    // judge result
+    std::string expectJson = R"({
+        "events" :
+        [
+            {
+                "contents" :
+                {
+                    "level": "ERROR",
+	                "msg": "expect { or n, but found \u0000, error found in #0 byte of ...",
+	                "time": "2024-07-04T06:59:23.078Z"
+                },
+                "timestamp" : 12345678901,
+                "timestampNanosecond" : 0,
+                "type" : 1
+            }
+        ]
+    })";
+    std::string outJson = eventGroupList[0].ToJsonString();
+    APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str());
+    APSARA_TEST_GT_FATAL(processorInstance.mProcTimeMS->GetValue(), uint64_t(0));
+}
+
 void ProcessorParseJsonNativeUnittest::TestProcessJson() {
     // make config
     Config config;