[log parts enhance] Use env to control ilogtail log level

CHANGELOG.md

@@ -40,6 +40,7 @@ your changes, such as:
 - [public] [both] [added] refactoried C++ process pipeline
 - [public] [both] [added] support use accelerate processors with go processors
 - [public] [both] [added] add new logtail metric module
+- [public] [both] [added] use env `ALIYUN_LOGTAIL_LOG_LEVEL` to control ilogtail log level
 - [public] [both] [updated] support continue/end regex patterns to split multiline log
 - [public] [both] [updated] support reader flush timeout
 - [public] [both] [updated] Flusher Kafka V2: support send the message with headers to kafka

core/common/StringTools.cpp

@@ -32,6 +32,12 @@ std::string ToLowerCaseString(const std::string& orig) {
     return copy;
 }
 
+std::string ToUpperCaseString(const std::string& orig) {
+    auto copy = orig;
+    std::transform(copy.begin(), copy.end(), copy.begin(), ::toupper);
+    return copy;
+}
+
 int StringCaseInsensitiveCmp(const std::string& s1, const std::string& s2) {
 #if defined(_MSC_VER)
     return _stricmp(s1.c_str(), s2.c_str());

core/common/StringTools.h

@@ -39,6 +39,7 @@ inline bool EndWith(const std::string& input, const std::string& pattern) {
 }
 
 std::string ToLowerCaseString(const std::string& orig);
+std::string ToUpperCaseString(const std::string& orig);
 
 int StringCaseInsensitiveCmp(const std::string& s1, const std::string& s2);
 int CStringNCaseInsensitiveCmp(const char* s1, const char* s2, size_t n);

core/logger/Logger.cpp

@@ -19,6 +19,7 @@
 #include <spdlog/sinks/rotating_file_sink.h>
 #include <spdlog/sinks/stdout_sinks.h>
 #include "common/RuntimeUtil.h"
+#include "common/StringTools.h"
 #include "common/FileSystemUtil.h"
 #include "common/Flags.h"
 #include "common/ErrorUtil.h"
@@ -267,6 +268,20 @@ void Logger::LoadConfig(const std::string& filePath) {
         logConfigInfo = "Load log config from " + filePath;
     } while (0);
 
+    // parse env log level
+    level::level_enum envLogLevel = level::info;
+    std::string aliyun_logtail_log_level;
+    const char* envLogLevelVal = std::getenv("ALIYUN_LOGTAIL_LOG_LEVEL");
+    if (envLogLevelVal) {
+        aliyun_logtail_log_level = envLogLevelVal;
+    }
+    if (MapStringToLevel(ToUpperCaseString(aliyun_logtail_log_level), envLogLevel)) {
+        LogMsg(std::string("Load log level from the env success, level: ") + aliyun_logtail_log_level);
+    } else {
+        LogMsg(std::string("Load log level from the env error, level: ") + aliyun_logtail_log_level);
+        aliyun_logtail_log_level = ""; 
+    }
+
     // Add or supply default config(s).
     bool needSave = true;
     if (loggerConfigs.empty()) {
@@ -305,7 +320,11 @@ void Logger::LoadConfig(const std::string& filePath) {
         spdlog::register_logger(logger);
         logger->set_level(loggerCfg.level);
         logger->set_pattern(DEFAULT_PATTERN);
-        logger->flush_on(loggerCfg.level);
+        if (name == "/apsara/sls/ilogtail" && !aliyun_logtail_log_level.empty()) {
+            logger->flush_on(envLogLevel);
+        } else {
+            logger->flush_on(loggerCfg.level);
+        }
         LogMsg(std::string("logger named ") + name + " created.");
     }
     if (failedLoggers.empty()) {
@@ -373,6 +392,7 @@ void Logger::SaveConfig(const std::string& filePath,
     out << Json::writeString(builder, rootVal);
 }
 
+
 void Logger::LoadDefaultConfig(std::map<std::string, LoggerConfig>& loggerCfgs,
                                std::map<std::string, SinkConfig>& sinkCfgs) {
     loggerCfgs.insert({DEFAULT_LOGGER_NAME, LoggerConfig{"AsyncFileSink", level::warn}});

docs/cn/configuration/system-config.md

@@ -33,5 +33,6 @@
 | `USE_CONTAINERD`       | Bool   | 是否使用containerd runtime，非必选。ilogtail会自动通过接口探测。                                                              |
 | `CONTAINERD_SOCK_PATH`       | String | 自定义containerd sock路径，非必选。默认为/run/containerd/containerd.sock。自定义取值可以通过查看/etc/containerd/config.toml grpc.address字段获取。 |
 | `CONTAINERD_STATE_DIR` | String | 自定义containerd 数据目录，非必选。自定义取值可以通过查看/etc/containerd/config.toml state字段获取。                                             |
+| `ALIYUN_LOGTAIL_LOG_LEVEL` | String |  用于控制/apsara/sls/ilogtail和golang插件的日志等级，支持通用日志等级，如trace, debug，info，warning，error，fatal|
 
 > 因为k8s本身自带资源限制的功能，所以如果你要将ilogtail部署到k8s中，可以通过将`cpu_usage_limit` 和 `mem_usage_limit` 设置为一个很大的值（比如99999999），以此来达到“关闭”ilogtail自身熔断功能的目的。

pkg/logger/logger.go

@@ -20,9 +20,10 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
+
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -268,10 +269,22 @@ func setLogConf(logConfig string) {
 	path := filepath.Clean(logConfig)
 	if _, err := os.Stat(path); err != nil {
 		logConfigContent := generateDefaultConfig()
-		_ = ioutil.WriteFile(path, []byte(logConfigContent), os.ModePerm)
+		_ = os.WriteFile(path, []byte(logConfigContent), os.ModePerm)
 	}
 	fmt.Fprintf(os.Stderr, "load log config %s \n", path)
-	logger, err := seelog.LoggerFromConfigAsFile(path)
+	content, err := os.ReadFile(path)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "init logger error", err)
+		return
+	}
+	dat := string(content)
+	aliyunLogtailLogLevel := strings.ToLower(os.Getenv("ALIYUN_LOGTAIL_LOG_LEVEL"))
+	if aliyunLogtailLogLevel != "" {
+		pattern := `(?mi)(minlevel=")([^"]*)(")`
+		regExp := regexp.MustCompile(pattern)
+		dat = regExp.ReplaceAllString(dat, `${1}`+aliyunLogtailLogLevel+`${3}`)
+	}
+	logger, err := seelog.LoggerFromConfigAsString(dat)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "init logger error", err)
 		return
@@ -281,8 +294,8 @@ func setLogConf(logConfig string) {
 		return
 	}
 	logtailLogger = logger
-	dat, _ := ioutil.ReadFile(path)
-	if strings.Contains(string(dat), "minlevel=\"debug\"") {
+
+	if aliyunLogtailLogLevel == "debug" || strings.Contains(dat, "minlevel=\"debug\"") {
 		debugFlag = 1
 	}
 }

pkg/logger/logger_test.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func clean() {
 }
 
 func readLog(index int) string {
-	bytes, _ := ioutil.ReadFile(util.GetCurrentBinaryPath() + "logtail_plugin.LOG")
+	bytes, _ := os.ReadFile(util.GetCurrentBinaryPath() + "logtail_plugin.LOG")
 	logs := strings.Split(string(bytes), "\n")
 	if index > len(logs)-1 {
 		return ""
@@ -189,6 +188,50 @@ func TestDebug(t *testing.T) {
 	}
 }
 
+func TestLogLevelFromEnv(t *testing.T) {
+	mu.Lock()
+	defer mu.Unlock()
+	clean()
+	os.Setenv("ALIYUN_LOGTAIL_LOG_LEVEL", "debug")
+	initNormalLogger()
+	os.Unsetenv("ALIYUN_LOGTAIL_LOG_LEVEL")
+	type args struct {
+		ctx     context.Context
+		kvPairs []interface{}
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "with-header",
+			args: args{
+				ctx:     ctx,
+				kvPairs: []interface{}{"a", "b"},
+			},
+			want: ".*\\[DBG\\] \\[logger_test.go:\\d{1,}\\] \\[func\\d{1,}\\] \\[mock-configname,mock-logstore\\]\t\\[a b\\]:.*",
+		},
+		{
+			name: "without-header",
+			args: args{
+				ctx:     context.Background(),
+				kvPairs: []interface{}{"a", "b"},
+			},
+			want: ".*\\[DBG\\] \\[logger_test.go:\\d{1,}\\] \\[func\\d{1,}\\] \\[a b\\]:.*",
+		},
+	}
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			Debug(tt.args.ctx, tt.args.kvPairs)
+			time.Sleep(time.Millisecond)
+			Flush()
+			log := readLog(i)
+			assert.True(t, regexp.MustCompile(tt.want).Match([]byte(log)), "want regexp %s, but got: %s", tt.want, log)
+		})
+	}
+}
+
 func TestDebugf(t *testing.T) {
 	mu.Lock()
 	defer mu.Unlock()