support V2 config

core/CMakeLists.txt

@@ -49,10 +49,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/common/Version.cpp.in ${VERSION_CPP_F
 if (UNIX)
     if (WITHOUTGDB)
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -fpic -fPIC -D_LARGEFILE64_SOURCE")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -fpic -fPIC -D_LARGEFILE64_SOURCE")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -fpic -fPIC -D_LARGEFILE64_SOURCE")
     else ()
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -g -ggdb -fpic -fPIC -D_LARGEFILE64_SOURCE")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -g -ggdb -fpic -fPIC -D_LARGEFILE64_SOURCE")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -g -ggdb -fpic -fPIC -D_LARGEFILE64_SOURCE")
     endif ()
     set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0")
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0")
@@ -86,8 +86,8 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/dependencies.cmake)
 
 # Subdirectories (modules).
 set(SUB_DIRECTORIES_LIST
-        aggregator app_config checkpoint common config config_manager config_server_pb
-        controller event event_handler event_listener file_server go_pipeline helper input log_pb logger models monitor
+        aggregator app_config checkpoint common config config_manager config_server_pb container_manager
+        controller event event_handler event_listener file_server flusher go_pipeline helper input log_pb logger models monitor
         parser pipeline plugin plugin/creator plugin/instance plugin/interface polling processor processor/daemon profile_sender reader sdk sender shennong sls_control
         fuse
         )

core/common/LogtailCommonFlags.cpp

@@ -15,15 +15,15 @@
 #include "LogtailCommonFlags.h"
 
 // Windows only has polling, give a bigger tail limit.
-#if defined(__linux__)
-DEFINE_FLAG_INT32(default_tail_limit_kb,
-                  "when first open file, if offset little than this value, move offset to beginning, KB",
-                  1024);
-#elif defined(_MSC_VER)
-DEFINE_FLAG_INT32(default_tail_limit_kb,
-                  "when first open file, if offset little than this value, move offset to beginning, KB",
-                  1024 * 50);
-#endif
+// #if defined(__linux__)
+// DEFINE_FLAG_INT32(default_tail_limit_kb,
+//                   "when first open file, if offset little than this value, move offset to beginning, KB",
+//                   1024);
+// #elif defined(_MSC_VER)
+// DEFINE_FLAG_INT32(default_tail_limit_kb,
+//                   "when first open file, if offset little than this value, move offset to beginning, KB",
+//                   1024 * 50);
+// #endif
 DEFINE_FLAG_INT32(monitor_interval, "program monitor interval, seconds", 30);
 DEFINE_FLAG_DOUBLE(cpu_usage_up_limit, "cpu usage upper limit, cores", 2.0);
 DEFINE_FLAG_DOUBLE(pub_cpu_usage_up_limit, "cpu usage upper limit, cores", 0.4);
@@ -71,7 +71,7 @@ DEFINE_FLAG_STRING(local_machine_uuid, "use this value if not empty, for ut/debu
 DEFINE_FLAG_STRING(user_defined_id_file, "", "user_defined_id");
 DEFINE_FLAG_STRING(logtail_sys_conf_users_dir, "", "users");
 DEFINE_FLAG_INT32(sls_client_send_timeout, "timeout time of one operation for SlsClient", 15);
-DEFINE_FLAG_BOOL(sls_client_send_compress, "whether compresses the data or not when put data", true);
+// DEFINE_FLAG_BOOL(sls_client_send_compress, "whether compresses the data or not when put data", true);
 DEFINE_FLAG_INT32(send_retrytimes, "how many times should retry if PostLogStoreLogs operation fail", 3);
 DEFINE_FLAG_INT32(default_StreamLog_tcp_port, "", 11111);
 DEFINE_FLAG_INT32(default_StreamLog_poll_size_in_mb, "", 50);

core/common/LogtailCommonFlags.h

@@ -26,7 +26,7 @@
 /////////////////////////////////////////////////////////////////////////
 
 DECLARE_FLAG_INT32(default_max_inotify_watch_num);
-DECLARE_FLAG_INT32(default_tail_limit_kb);
+// DECLARE_FLAG_INT32(default_tail_limit_kb);
 DECLARE_FLAG_STRING(logtail_send_address);
 DECLARE_FLAG_STRING(logtail_config_address);
 DECLARE_FLAG_INT32(cpu_limit_num);
@@ -56,7 +56,7 @@ DECLARE_FLAG_STRING(default_aliuid);
 DECLARE_FLAG_STRING(default_access_key_id);
 DECLARE_FLAG_STRING(default_access_key);
 DECLARE_FLAG_INT32(sls_client_send_timeout);
-DECLARE_FLAG_BOOL(sls_client_send_compress);
+// DECLARE_FLAG_BOOL(sls_client_send_compress);
 DECLARE_FLAG_INT32(send_retrytimes);
 DECLARE_FLAG_STRING(user_log_config);
 DECLARE_FLAG_STRING(ilogtail_config);

core/common/ParamExtractor.cpp

@@ -0,0 +1,142 @@
+// Copyright 2023 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/ParamExtractor.h"
+
+#include "boost/regex.hpp"
+
+using namespace std;
+
+namespace logtail {
+
+string ExtractCurrentKey(const string& key) {
+    size_t pos = key.rfind('.');
+    if (pos == string::npos) {
+        return key;
+    }
+    return key.substr(pos + 1);
+}
+
+bool GetOptionalBoolParam(const Json::Value& config, const string& key, bool& param, string& errorMsg) {
+    errorMsg.clear();
+    string curKey = ExtractCurrentKey(key);
+    const Json::Value* itr = config.find(curKey.c_str(), curKey.c_str() + curKey.length());
+    if (itr != nullptr) {
+        if (!itr->isBool()) {
+            errorMsg = "param " + key + " is not of type bool";
+            return false;
+        }
+        param = itr->asBool();
+    }
+    return true;
+}
+
+bool GetOptionalIntParam(const Json::Value& config, const string& key, int32_t& param, string& errorMsg) {
+    errorMsg.clear();
+    string curKey = ExtractCurrentKey(key);
+    const Json::Value* itr = config.find(curKey.c_str(), curKey.c_str() + curKey.length());
+    if (itr != nullptr) {
+        if (!itr->isInt()) {
+            errorMsg = "param " + key + " is not of type int";
+            return false;
+        }
+        param = itr->asInt();
+    }
+    return true;
+}
+
+bool GetOptionalUIntParam(const Json::Value& config, const string& key, uint32_t& param, string& errorMsg) {
+    errorMsg.clear();
+    string curKey = ExtractCurrentKey(key);
+    const Json::Value* itr = config.find(curKey.c_str(), curKey.c_str() + curKey.length());
+    if (itr != nullptr) {
+        if (!itr->isUInt()) {
+            errorMsg = "param " + key + " is not of type uint";
+            return false;
+        }
+        param = itr->asUInt();
+    }
+    return true;
+}
+
+bool GetOptionalStringParam(const Json::Value& config, const string& key, string& param, string& errorMsg) {
+    errorMsg.clear();
+    string curKey = ExtractCurrentKey(key);
+    const Json::Value* itr = config.find(curKey.c_str(), curKey.c_str() + curKey.length());
+    if (itr != nullptr) {
+        if (!itr->isString()) {
+            errorMsg = "param " + key + " is not of type string";
+            return false;
+        }
+        param = itr->asString();
+    }
+    return true;
+}
+
+bool GetMandatoryBoolParam(const Json::Value& config, const string& key, bool& param, string& errorMsg) {
+    errorMsg.clear();
+    if (!config.isMember(ExtractCurrentKey(key))) {
+        errorMsg = "madatory param " + key + " is missing";
+        return false;
+    }
+    return GetOptionalBoolParam(config, key, param, errorMsg);
+}
+
+bool GetMandatoryIntParam(const Json::Value& config, const string& key, int32_t& param, string& errorMsg) {
+    errorMsg.clear();
+    if (!config.isMember(ExtractCurrentKey(key))) {
+        errorMsg = "madatory param " + key + " is missing";
+        return false;
+    }
+    return GetOptionalIntParam(config, key, param, errorMsg);
+}
+
+bool GetMandatoryUIntParam(const Json::Value& config, const string& key, uint32_t& param, string& errorMsg) {
+    errorMsg.clear();
+    if (!config.isMember(ExtractCurrentKey(key))) {
+        errorMsg = "madatory param " + key + " is missing";
+        return false;
+    }
+    return GetOptionalUIntParam(config, key, param, errorMsg);
+}
+
+bool GetMandatoryStringParam(const Json::Value& config, const string& key, string& param, string& errorMsg) {
+    errorMsg.clear();
+    if (!config.isMember(ExtractCurrentKey(key))) {
+        errorMsg = "madatory param " + key + " is missing";
+        return false;
+    }
+    if (!GetOptionalStringParam(config, key, param, errorMsg)) {
+        return false;
+    }
+    if (param.empty()) {
+        errorMsg = "madatory string param " + key + " is empty";
+        return false;
+    }
+    return true;
+}
+
+bool IsRegexValid(const string& regStr) {
+    if (regStr.empty()) {
+        return true;
+    }
+    try {
+        boost::regex reg(regStr);
+    } catch (...) {
+        return false;
+    }
+    return true;
+}
+
+} // namespace logtail