[log parts enhance] Use env to control ilogtail log level

core/common/StringTools.cpp

@@ -16,6 +16,7 @@
 #include <string.h>
 #include <boost/algorithm/string.hpp>
 #include <boost/xpressive/xpressive.hpp>
+#include <cctype>
 #include "logger/Logger.h"
 #if defined(_MSC_VER)
 #include <Shlwapi.h>
@@ -32,6 +33,12 @@ std::string ToLowerCaseString(const std::string& orig) {
     return copy;
 }
 
+std::string ToUpperCaseString(const std::string& orig) {
+    auto copy = orig;
+    std::transform(copy.begin(), copy.end(), copy.begin(), ::toupper);
+    return copy;
+}
+
 int StringCaseInsensitiveCmp(const std::string& s1, const std::string& s2) {
 #if defined(_MSC_VER)
     return _stricmp(s1.c_str(), s2.c_str());

core/common/StringTools.h

@@ -39,6 +39,7 @@ inline bool EndWith(const std::string& input, const std::string& pattern) {
 }
 
 std::string ToLowerCaseString(const std::string& orig);
+std::string ToUpperCaseString(const std::string& orig);
 
 int StringCaseInsensitiveCmp(const std::string& s1, const std::string& s2);
 int CStringNCaseInsensitiveCmp(const char* s1, const char* s2, size_t n);

core/logger/Logger.cpp

@@ -19,6 +19,7 @@
 #include <spdlog/sinks/rotating_file_sink.h>
 #include <spdlog/sinks/stdout_sinks.h>
 #include "common/RuntimeUtil.h"
+#include "common/StringTools.h"
 #include "common/FileSystemUtil.h"
 #include "common/Flags.h"
 #include "common/ErrorUtil.h"
@@ -267,14 +268,25 @@ void Logger::LoadConfig(const std::string& filePath) {
         logConfigInfo = "Load log config from " + filePath;
     } while (0);
 
+    // parse env log level
+    level::level_enum* envLogLevel = new(level::level_enum);
+    std::string aliyun_logtail_log_level = std::getenv("ALIYUN_LOGTAIL_LOG_LEVEL");
+    std::string logLevelInfo;
+    if (MapStringToLevel(ToUpperCaseString(aliyun_logtail_log_level), *envLogLevel)) {
+        logLevelInfo = "Load log level from the env success, level: " + aliyun_logtail_log_level;
+    } else {
+        logLevelInfo = "Load log level from the env error, level: " + aliyun_logtail_log_level;
+    }
+    LogMsg(logLevelInfo);
+
     // Add or supply default config(s).
     bool needSave = true;
     if (loggerConfigs.empty()) {
         logConfigInfo = "Load log config file failed, use default configs.";
-        LoadAllDefaultConfigs(loggerConfigs, sinkConfigs);
+        LoadAllDefaultConfigs(loggerConfigs, sinkConfigs, envLogLevel);
     } else if (loggerConfigs.end() == loggerConfigs.find(DEFAULT_LOGGER_NAME)) {
         logConfigInfo += ", and add default config.";
-        LoadDefaultConfig(loggerConfigs, sinkConfigs);
+        LoadDefaultConfig(loggerConfigs, sinkConfigs, envLogLevel);
     } else
         needSave = false;
 
@@ -305,7 +317,11 @@ void Logger::LoadConfig(const std::string& filePath) {
         spdlog::register_logger(logger);
         logger->set_level(loggerCfg.level);
         logger->set_pattern(DEFAULT_PATTERN);
-        logger->flush_on(loggerCfg.level);
+        if (name == "/apsara/sls/ilogtail" && envLogLevel) {
+            logger->flush_on(*envLogLevel);
+        } else {
+            logger->flush_on(loggerCfg.level);
+        }
         LogMsg(std::string("logger named ") + name + " created.");
     }
     if (failedLoggers.empty()) {
@@ -373,22 +389,33 @@ void Logger::SaveConfig(const std::string& filePath,
     out << Json::writeString(builder, rootVal);
 }
 
+
 void Logger::LoadDefaultConfig(std::map<std::string, LoggerConfig>& loggerCfgs,
-                               std::map<std::string, SinkConfig>& sinkCfgs) {
-    loggerCfgs.insert({DEFAULT_LOGGER_NAME, LoggerConfig{"AsyncFileSink", level::warn}});
+                               std::map<std::string, SinkConfig>& sinkCfgs,
+                               level::level_enum* envLogLevel) {
+    level::level_enum logLevel = level::warn;
+    if (envLogLevel) {
+        logLevel = *envLogLevel;
+    }
+    loggerCfgs.insert({DEFAULT_LOGGER_NAME, LoggerConfig{"AsyncFileSink", logLevel}});
     if (sinkCfgs.find("AsyncFileSink") != sinkCfgs.end())
         return;
     sinkCfgs.insert({"AsyncFileSink",
                      SinkConfig{"AsyncFile", 10, 20000000, 300, GetProcessExecutionDir() + "ilogtail.LOG", "Gzip"}});
 }
 
 void Logger::LoadAllDefaultConfigs(std::map<std::string, LoggerConfig>& loggerCfgs,
-                                   std::map<std::string, SinkConfig>& sinkCfgs) {
-    LoadDefaultConfig(loggerCfgs, sinkCfgs);
+                                   std::map<std::string, SinkConfig>& sinkCfgs,
+                                   level::level_enum* envLogLevel) {
+    LoadDefaultConfig(loggerCfgs, sinkCfgs, envLogLevel);
 
-    loggerCfgs.insert({"/apsara/sls/ilogtail", LoggerConfig{"AsyncFileSink", level::info}});
-    loggerCfgs.insert({"/apsara/sls/ilogtail/profile", LoggerConfig{"AsyncFileSinkProfile", level::info}});
-    loggerCfgs.insert({"/apsara/sls/ilogtail/status", LoggerConfig{"AsyncFileSinkStatus", level::info}});
+    level::level_enum logLevel = level::info;
+    if (envLogLevel) {
+        logLevel = *envLogLevel;
+    }
+    loggerCfgs.insert({"/apsara/sls/ilogtail", LoggerConfig{"AsyncFileSink", logLevel}});
+    loggerCfgs.insert({"/apsara/sls/ilogtail/profile", LoggerConfig{"AsyncFileSinkProfile", logLevel}});
+    loggerCfgs.insert({"/apsara/sls/ilogtail/status", LoggerConfig{"AsyncFileSinkStatus", logLevel}});
 
     std::string dirPath = GetProcessExecutionDir() + STRING_FLAG(logtail_snapshot_dir);
     if (!Mkdir(dirPath)) {

core/logger/Logger.h

@@ -75,10 +75,12 @@ class Logger {
 
     // LoadDefaultConfig only loads the default ("/") into @loggerCfgs and @sinkCfgs.
     void LoadDefaultConfig(std::map<std::string, LoggerConfig>& loggerCfgs,
-                           std::map<std::string, SinkConfig>& sinkCfgs);
+                           std::map<std::string, SinkConfig>& sinkCfgs,
+                           spdlog::level::level_enum* envLogLvel);
     // LoadAllDefaultConfigs loads all default configs into @loggerCfgs and @sinkCfgs.
     void LoadAllDefaultConfigs(std::map<std::string, LoggerConfig>& loggerCfgs,
-                               std::map<std::string, SinkConfig>& sinkCfgs);
+                               std::map<std::string, SinkConfig>& sinkCfgs,
+                               spdlog::level::level_enum* envLogLvel);
 
     // EnsureSnapshotDirExist ensures the snapshot dir exists.
     void EnsureSnapshotDirExist(std::map<std::string, SinkConfig>& sinkCfgs);

pkg/logger/logger.go

@@ -23,6 +23,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -271,7 +272,19 @@ func setLogConf(logConfig string) {
 		_ = ioutil.WriteFile(path, []byte(logConfigContent), os.ModePerm)
 	}
 	fmt.Fprintf(os.Stderr, "load log config %s \n", path)
-	logger, err := seelog.LoggerFromConfigAsFile(path)
+	content, err := os.ReadFile(path)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "init logger error", err)
+		return
+	}
+	dat := string(content)
+	aliyun_logtail_log_level := strings.ToLower(os.Getenv("ALIYUN_LOGTAIL_LOG_LEVEL"))
+	if aliyun_logtail_log_level != "" {
+		pattern := `(?mi)(<seelog\s+[^>]*\bminlevel=")[^"]*(("[^>]*>)|("))`
+		regExp := regexp.MustCompile(pattern)
+		dat = regExp.ReplaceAllString(dat, `${1}`+aliyun_logtail_log_level+`${3}`)
+	}
+	logger, err := seelog.LoggerFromConfigAsString(dat)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "init logger error", err)
 		return
@@ -281,8 +294,10 @@ func setLogConf(logConfig string) {
 		return
 	}
 	logtailLogger = logger
-	dat, _ := ioutil.ReadFile(path)
-	if strings.Contains(string(dat), "minlevel=\"debug\"") {
+
+	if aliyun_logtail_log_level == "debug" {
+		debugFlag = 1
+	} else if strings.Contains(dat, "minlevel=\"debug\"") {
 		debugFlag = 1
 	}
 }

pkg/logger/logger_test.go

@@ -189,6 +189,51 @@ func TestDebug(t *testing.T) {
 	}
 }
 
+func TestLogLevelFromEnv(t *testing.T) {
+	mu.Lock()
+	defer mu.Unlock()
+	flag.Set(FlagLevelName, seelog.DebugStr)
+	clean()
+	os.Setenv("ALIYUN_LOGTAIL_LOG_LEVEL", "info")
+	initNormalLogger()
+	os.Unsetenv("ALIYUN_LOGTAIL_LOG_LEVEL")
+	type args struct {
+		ctx     context.Context
+		kvPairs []interface{}
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "with-header",
+			args: args{
+				ctx:     ctx,
+				kvPairs: []interface{}{"a", "b"},
+			},
+			want: ".*\\[INF\\] \\[logger_test.go:\\d{1,}\\] \\[func\\d{1,}\\] \\[mock-configname,mock-logstore\\]\t\\[a b\\]:.*",
+		},
+		{
+			name: "without-header",
+			args: args{
+				ctx:     context.Background(),
+				kvPairs: []interface{}{"a", "b"},
+			},
+			want: ".*\\[INF\\] \\[logger_test.go:\\d{1,}\\] \\[func\\d{1,}\\] \\[a b\\]:.*",
+		},
+	}
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			Info(tt.args.ctx, tt.args.kvPairs)
+			time.Sleep(time.Millisecond)
+			Flush()
+			log := readLog(i)
+			assert.True(t, regexp.MustCompile(tt.want).Match([]byte(log)), "want regexp %s, but got: %s", tt.want, log)
+		})
+	}
+}
+
 func TestDebugf(t *testing.T) {
 	mu.Lock()
 	defer mu.Unlock()