Merge remote-tracking branch 'github/main' into fix/forceReleaseStopC…

…ontainerFile_main
alibaba · Jan 23, 2025 · 76bbdad · 76bbdad
2 parents 54efb8e + 183af3a
commit 76bbdad
Show file tree

Hide file tree

Showing 385 changed files with 6,418 additions and 2,789 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector-build-linux:2.1.1
+FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector-build-linux:2.1.2
 
 ARG USERNAME=admin
 ARG USER_PASSWORD
@@ -40,11 +40,13 @@ RUN source /tmp/.env && rm /tmp/.env; \
     fi; \
     useradd --uid $USER_UID --gid $GROUP_GID -m $USERNAME; \
     if [ -n "$USER_PASSWORD" ]; then echo "$USERNAME:$USER_PASSWORD" | chpasswd; fi; \
-    echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME; \
-    chmod 0440 /etc/sudoers.d/$USERNAME; \
+    # the filename should not have the . or ~ symbol.
+    USER_SUDOER_FILE=$(echo $USERNAME | sed 's/\./-/g'); \
+    echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USER_SUDOER_FILE; \
+    chmod 0440 /etc/sudoers.d/$USER_SUDOER_FILE; \
     chown -R $USERNAME:$GROUPNAME /opt $(eval echo ~$USERNAME); \
     chmod -R 755 $(eval echo ~$USERNAME);
 
 USER $USERNAME
 
-RUN go env -w GO111MODULE=on && go env -w GOPROXY=https://goproxy.cn,direct
+RUN go env -w GO111MODULE=on && go env -w GOPROXY=https://goproxy.cn,direct
diff --git a/README.md b/README.md
@@ -94,7 +94,7 @@ You can report bugs, make suggestions or participate in discussions through [Git
 * Zhihu：[LoongCollector社区](https://www.zhihu.com/column/c_1533139823409270785)
 * DingTalk/WeChat：LoongCollector社区
 
-<img src="docs/cn/.gitbook/assets/chatgroup.png" style="width: 60%; height: 60%" />
+<img src="https://ilogtail-community-edition.oss-cn-shanghai.aliyuncs.com/images/chatgroup/chatgroup.png" style="width: 60%; height: 60%" />
 
 ## Our Users
 

diff --git a/config_server/service/Dockerfile b/config_server/service/Dockerfile
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector-build-linux:2.0.5 as build
+FROM sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector-build-linux:2.1.2 as build
 
 USER root
 WORKDIR /src
@@ -21,7 +21,7 @@ RUN go env -w GOPROXY="https://goproxy.cn,direct" && \
     go build -o ConfigServer
 
 FROM centos:centos7.9.2009
-MAINTAINER TomYu [email protected]
+LABEL maintainer="TomYu <[email protected]>"
 
 ENV container docker
 

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
@@ -121,7 +121,7 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/plugin/flusher/links.cmake)
 set(SUB_DIRECTORIES_LIST
         application app_config checkpoint container_manager metadata logger go_pipeline monitor monitor/metric_constants monitor/metric_models monitor/profile_sender models
         config config/watcher constants
-        pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer
+        collection_pipeline collection_pipeline/batch collection_pipeline/limiter collection_pipeline/plugin collection_pipeline/plugin/creator collection_pipeline/plugin/instance collection_pipeline/plugin/interface collection_pipeline/queue collection_pipeline/route collection_pipeline/serializer
         task_pipeline
         runner runner/sink/http
         protobuf/sls protobuf/models

diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp
@@ -17,12 +17,14 @@
 #include <algorithm>
 #include <filesystem>
 #include <iostream>
+#include <unordered_set>
 #include <utility>
 
 #include "boost/filesystem.hpp"
 #include "json/value.h"
 
 #include "RuntimeUtil.h"
+#include "StringTools.h"
 #include "common/EnvUtil.h"
 #include "common/FileSystemUtil.h"
 #include "common/JsonUtil.h"
@@ -161,9 +163,9 @@ DEFINE_FLAG_STRING(metrics_report_method,
                    "method to report metrics (default none, means logtail will not report metrics)",
                    "sls");
 
-DEFINE_FLAG_STRING(loong_collector_operator_service, "loong collector operator service", "");
-DEFINE_FLAG_INT32(loong_collector_operator_service_port, "loong collector operator service port", 8888);
-DEFINE_FLAG_INT32(loong_collector_k8s_meta_service_port, "loong collector operator service port", 9000);
+DEFINE_FLAG_STRING(operator_service, "loong collector operator service", "");
+DEFINE_FLAG_INT32(operator_service_port, "loong collector operator service port", 8888);
+DEFINE_FLAG_INT32(k8s_meta_service_port, "loong collector operator service port", 9000);
 DEFINE_FLAG_STRING(_pod_name_, "agent pod name", "");
 
 DEFINE_FLAG_STRING(app_info_file, "", "app_info.json");
@@ -210,14 +212,21 @@ const uint32_t NO_FALL_BACK_FAIL_PERCENTAGE = 10;
 const uint32_t SLOW_FALL_BACK_FAIL_PERCENTAGE = 40;
 
 std::string AppConfig::sLocalConfigDir = "local";
+
+const std::string LOONGCOLLECTOR_ENV_PREFIX = "LOONG_";
+
+std::string GetLoongcollectorEnv(const std::string& flagName) {
+    return LOONGCOLLECTOR_ENV_PREFIX + ToUpperCaseString(flagName);
+}
+
 void CreateAgentDir() {
     try {
-        const char* value = getenv("logtail_mode");
+        const char* value = getenv("LOGTAIL_MODE");
         if (value != NULL) {
             STRING_FLAG(logtail_mode) = StringTo<bool>(value);
         }
     } catch (const exception& e) {
-        std::cout << "load config from env error, env_name:logtail_mode, error:" << e.what() << std::endl;
+        std::cout << "load config from env error, env_name:LOGTAIL_MODE, error:" << e.what() << std::endl;
     }
     if (BOOL_FLAG(logtail_mode)) {
         return;
@@ -226,7 +235,8 @@ void CreateAgentDir() {
     Json::Value emptyJson;
 #define PROCESSDIRFLAG(flag_name) \
     try { \
-        const char* value = getenv(#flag_name); \
+        const auto env_name = GetLoongcollectorEnv(#flag_name); \
+        const char* value = getenv(env_name.c_str()); \
         if (value != NULL) { \
             STRING_FLAG(flag_name) = StringTo<string>(value); \
         } \
@@ -247,11 +257,11 @@ void CreateAgentDir() {
         } \
     }
 
-    PROCESSDIRFLAG(loongcollector_conf_dir);
-    PROCESSDIRFLAG(loongcollector_log_dir);
-    PROCESSDIRFLAG(loongcollector_data_dir);
-    PROCESSDIRFLAG(loongcollector_run_dir);
-    PROCESSDIRFLAG(loongcollector_third_party_dir);
+    PROCESSDIRFLAG(conf_dir);
+    PROCESSDIRFLAG(logs_dir);
+    PROCESSDIRFLAG(data_dir);
+    PROCESSDIRFLAG(run_dir);
+    PROCESSDIRFLAG(third_party_dir);
 }
 
 std::string GetAgentThirdPartyDir() {
@@ -262,7 +272,7 @@ std::string GetAgentThirdPartyDir() {
     if (BOOL_FLAG(logtail_mode)) {
         dir = AppConfig::GetInstance()->GetLoongcollectorConfDir();
     } else {
-        dir = STRING_FLAG(loongcollector_third_party_dir) + PATH_SEPARATOR;
+        dir = STRING_FLAG(third_party_dir) + PATH_SEPARATOR;
     }
     return dir;
 }
@@ -278,7 +288,7 @@ std::string GetAgentLogDir() {
     if (BOOL_FLAG(logtail_mode)) {
         dir = GetProcessExecutionDir();
     } else {
-        dir = STRING_FLAG(loongcollector_log_dir) + PATH_SEPARATOR;
+        dir = STRING_FLAG(logs_dir) + PATH_SEPARATOR;
     }
 #endif
     return dir;
@@ -370,9 +380,9 @@ std::string GetAgentDataDir() {
     dir = GetProcessExecutionDir();
 #else
     if (BOOL_FLAG(logtail_mode)) {
-        dir = AppConfig::GetInstance()->GetLoongcollectorConfDir() + PATH_SEPARATOR + "checkpoint";
+        dir = AppConfig::GetInstance()->GetLoongcollectorConfDir() + PATH_SEPARATOR + "checkpoint" + PATH_SEPARATOR;
     } else {
-        dir = STRING_FLAG(loongcollector_data_dir) + PATH_SEPARATOR;
+        dir = STRING_FLAG(data_dir) + PATH_SEPARATOR;
     }
 #endif
     if (!CheckExistance(dir)) {
@@ -396,7 +406,7 @@ std::string GetAgentConfDir() {
     if (BOOL_FLAG(logtail_mode)) {
         dir = GetProcessExecutionDir();
     } else {
-        dir = STRING_FLAG(loongcollector_conf_dir) + PATH_SEPARATOR;
+        dir = STRING_FLAG(conf_dir) + PATH_SEPARATOR;
     }
 #endif
     return dir;
@@ -413,7 +423,7 @@ std::string GetAgentRunDir() {
     if (BOOL_FLAG(logtail_mode)) {
         dir = GetProcessExecutionDir();
     } else {
-        dir = STRING_FLAG(loongcollector_run_dir) + PATH_SEPARATOR;
+        dir = STRING_FLAG(run_dir) + PATH_SEPARATOR;
     }
 #endif
     return dir;
@@ -838,28 +848,22 @@ bool LoadSingleValueEnvConfig(const char* envKey, T& configValue, const T minVal
     } catch (const exception& e) {
         LOG_WARNING(sLogger, (string("set ") + envKey + " from env failed, exception", e.what()));
     }
-    return false;
-}
-
-/**
- * @brief 从环境变量加载配置值（如果存在）
- *
- * @tparam T 配置值的类型
- * @param envKey 环境变量的键
- * @param cfgValue 配置值的引用，如果环境变量存在，将被更新
- */
-template <typename T>
-void LoadEnvValueIfExisting(const char* envKey, T& cfgValue) {
     try {
-        const char* value = getenv(envKey);
+        char* value = NULL;
+        const auto newEnvKey = LOONGCOLLECTOR_ENV_PREFIX + ToUpperCaseString(envKey);
+        value = getenv(newEnvKey.c_str());
         if (value != NULL) {
             T val = StringTo<T>(value);
-            cfgValue = val;
-            LOG_INFO(sLogger, ("load config from env", envKey)("value", val));
+            if (val >= minValue) {
+                configValue = val;
+                LOG_INFO(sLogger, (string("set ") + envKey + " from env, value", value));
+                return true;
+            }
         }
-    } catch (const std::exception& e) {
-        LOG_WARNING(sLogger, ("load config from env error", envKey)("error", e.what()));
+    } catch (const exception& e) {
+        LOG_WARNING(sLogger, (string("set ") + envKey + " from env failed, exception", e.what()));
     }
+    return false;
 }
 
 void AppConfig::LoadEnvResourceLimit() {
@@ -1457,11 +1461,7 @@ void AppConfig::InitEnvMapping(const std::string& envStr, std::map<std::string,
     }
 }
 void AppConfig::SetConfigFlag(const std::string& flagName, const std::string& value) {
-    static set<string> sIgnoreFlagSet = {"loongcollector_conf_dir",
-                                         "loongcollector_log_dir",
-                                         "loongcollector_data_dir",
-                                         "loongcollector_run_dir",
-                                         "logtail_mode"};
+    static set<string> sIgnoreFlagSet = {"conf_dir", "logs_dir", "data_dir", "run_dir", "logtail_mode"};
     if (sIgnoreFlagSet.find(flagName) != sIgnoreFlagSet.end()) {
         return;
     }
@@ -1508,9 +1508,43 @@ void AppConfig::ParseEnvToFlags() {
         }
     }
 #endif
+    std::unordered_set<std::string> sIgnoreFlagSet = {"buffer_file_path",
+                                                      "check_point_filename",
+                                                      "data_server_port",
+                                                      "host_path_blacklist",
+                                                      "process_thread_count",
+                                                      "send_request_concurrency",
+                                                      "check_point_dump_interval",
+                                                      "check_point_max_count",
+                                                      "enable_root_path_collection",
+                                                      "ilogtail_config",
+                                                      "ilogtail_discard_interval",
+                                                      "default_tail_limit_kb",
+                                                      "logreader_max_rotate_queue_size",
+                                                      "force_release_deleted_file_fd_timeout",
+                                                      "batch_send_interval",
+                                                      "ALIYUN_LOG_FILE_TAGS",
+                                                      "default_container_host_path",
+                                                      "default_max_inotify_watch_num",
+                                                      "enable_full_drain_mode",
+                                                      "ilogtail_discard_old_data",
+                                                      "timeout_interval",
+                                                      "enable_env_ref_in_config",
+                                                      "max_watch_dir_count",
+                                                      "polling_max_stat_count",
+                                                      "polling_max_stat_count_per_config",
+                                                      "polling_max_stat_count_per_dir"};
     for (const auto& iter : envMapping) {
-        const std::string& key = iter.first;
         const std::string& value = iter.second;
+        std::string key = iter.first;
+        // Skip if key is not in ignore set and doesn't start with prefix
+        if (sIgnoreFlagSet.find(key) == sIgnoreFlagSet.end() && !StartWith(key, LOONGCOLLECTOR_ENV_PREFIX)) {
+            continue;
+        }
+        // Convert to lowercase if key has prefix
+        if (StartWith(key, LOONGCOLLECTOR_ENV_PREFIX)) {
+            key = ToLowerCaseString(key.substr(LOONGCOLLECTOR_ENV_PREFIX.size()));
+        }
         SetConfigFlag(key, value);
         // 尝试解析为 double
         char* end;

diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h
@@ -38,6 +38,7 @@ extern const uint32_t CONCURRENCY_STATISTIC_THRESHOLD;
 extern const uint32_t CONCURRENCY_STATISTIC_INTERVAL_THRESHOLD_SECONDS;
 extern const uint32_t NO_FALL_BACK_FAIL_PERCENTAGE;
 extern const uint32_t SLOW_FALL_BACK_FAIL_PERCENTAGE;
+extern const std::string LOONGCOLLECTOR_ENV_PREFIX;
 
 void CreateAgentDir();
 
@@ -74,6 +75,7 @@ std::string GetAgentName();
 std::string GetMonitorInfoFileName();
 std::string GetSymLinkName();
 std::string GetAgentPrefix();
+std::string GetFileTagsDir();
 
 template <class T>
 class DoubleBuffer {
@@ -542,6 +544,7 @@ class AppConfig {
     friend class EnterpriseSLSClientManagerUnittest;
     friend class FlusherRunnerUnittest;
     friend class PipelineUpdateUnittest;
+    friend class ProcessorTagNativeUnittest;
 #endif
 };
 

diff --git a/core/application/Application.cpp b/core/application/Application.cpp
@@ -22,6 +22,10 @@
 
 #include "app_config/AppConfig.h"
 #include "checkpoint/CheckPointManager.h"
+#include "collection_pipeline/CollectionPipelineManager.h"
+#include "collection_pipeline/plugin/PluginRegistry.h"
+#include "collection_pipeline/queue/ExactlyOnceQueueManager.h"
+#include "collection_pipeline/queue/SenderQueueManager.h"
 #include "common/CrashBackTraceUtil.h"
 #include "common/Flags.h"
 #include "common/MachineInfoUtil.h"
@@ -41,10 +45,6 @@
 #include "go_pipeline/LogtailPlugin.h"
 #include "logger/Logger.h"
 #include "monitor/Monitor.h"
-#include "pipeline/PipelineManager.h"
-#include "pipeline/plugin/PluginRegistry.h"
-#include "pipeline/queue/ExactlyOnceQueueManager.h"
-#include "pipeline/queue/SenderQueueManager.h"
 #include "plugin/flusher/sls/DiskBufferWriter.h"
 #include "plugin/flusher/sls/FlusherSLS.h"
 #include "plugin/input/InputFeedbackInterfaceRegistry.h"
@@ -65,7 +65,7 @@
 #endif
 
 DEFINE_FLAG_BOOL(ilogtail_disable_core, "disable core in worker process", true);
-DEFINE_FLAG_INT32(file_tags_update_interval, "second", 1);
+DEFINE_FLAG_INT32(file_tags_update_interval, "second", 60);
 DEFINE_FLAG_INT32(config_scan_interval, "seconds", 10);
 DEFINE_FLAG_INT32(tcmalloc_release_memory_interval, "force release memory held by tcmalloc, seconds", 300);
 DEFINE_FLAG_INT32(exit_flushout_duration, "exit process flushout duration", 20 * 1000);
@@ -284,7 +284,7 @@ void Application::Start() { // GCOVR_EXCL_START
         if (curTime - lastConfigCheckTime >= INT32_FLAG(config_scan_interval)) {
             auto configDiff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff();
             if (!configDiff.first.IsEmpty()) {
-                PipelineManager::GetInstance()->UpdatePipelines(configDiff.first);
+                CollectionPipelineManager::GetInstance()->UpdatePipelines(configDiff.first);
             }
             if (!configDiff.second.IsEmpty()) {
                 TaskPipelineManager::GetInstance()->UpdatePipelines(configDiff.second);
@@ -357,7 +357,7 @@ void Application::Exit() {
     }
 #endif
 
-    PipelineManager::GetInstance()->StopAllPipelines();
+    CollectionPipelineManager::GetInstance()->StopAllPipelines();
 
     PluginRegistry::GetInstance()->UnloadPlugins();
 

diff --git a/core/checkpoint/RangeCheckpoint.h b/core/checkpoint/RangeCheckpoint.h
@@ -19,7 +19,7 @@
 #include <string>
 #include <vector>
 
-#include "pipeline/queue/QueueKey.h"
+#include "collection_pipeline/queue/QueueKey.h"
 #include "protobuf/sls/checkpoint.pb.h"
 
 namespace logtail {