Upgrade environment variable logic using new Pipeline API for logtail environment configuration management

CHANGELOG.md

@@ -38,3 +38,4 @@ your changes, such as:
 ## [Unreleased]
 
 - [public] [both] [added] add UsingOldContentTag. When UsingOldContentTag is set to false, the Tag is now placed in the Meta instead of Logs during cgo.
+- [public] [both] [added] Env control using the new configuration

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/VictoriaMetrics/VictoriaMetrics v1.83.1
 	github.com/alibaba/ilogtail/pkg v0.0.0
 	github.com/aliyun/alibaba-cloud-sdk-go/services/sls_inner v0.0.0
-	github.com/aliyun/aliyun-log-go-sdk v0.1.37
 	github.com/apache/pulsar-client-go v0.10.0
 	github.com/buger/jsonparser v1.1.1
 	github.com/cespare/xxhash v1.1.0 // indirect
@@ -56,7 +55,7 @@ require (
 	go.opentelemetry.io/collector/pdata v0.66.0
 	go.opentelemetry.io/proto/otlp v0.19.0
 	go.uber.org/atomic v1.10.0
-	golang.org/x/sys v0.6.0
+	golang.org/x/sys v0.9.0
 	google.golang.org/grpc v1.53.0
 	google.golang.org/protobuf v1.31.0
 	gotest.tools v2.2.0+incompatible
@@ -152,7 +151,6 @@ require (
 	github.com/Microsoft/hcsshim v0.9.4 // indirect
 	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483
 	github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 // indirect
-	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/containerd/cgroups v1.0.4 // indirect
 	github.com/containerd/containerd v1.6.8 // indirect
 	github.com/containerd/continuity v0.2.2 // indirect
@@ -190,7 +188,7 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect
+	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
@@ -237,12 +235,12 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/multierr v1.9.0
 	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/crypto v0.1.0 // indirect
-	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/crypto v0.10.0 // indirect
+	golang.org/x/net v0.11.0 // indirect
 	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/term v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/term v0.9.0 // indirect
+	golang.org/x/text v0.10.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect
@@ -284,7 +282,30 @@ require (
 	sigs.k8s.io/gateway-api v0.6.2 // indirect
 )
 
-require github.com/rogpeppe/go-internal v1.6.1 // indirect
+require (
+	github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.5
+	github.com/alibabacloud-go/sls-20201230/v5 v5.1.2
+	github.com/alibabacloud-go/tea v1.2.1
+)
+
+require (
+	github.com/alibabacloud-go/alibabacloud-gateway-sls v0.0.6 // indirect
+	github.com/alibabacloud-go/alibabacloud-gateway-sls-util v0.0.1 // indirect
+	github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect
+	github.com/alibabacloud-go/darabonba-array v0.1.0 // indirect
+	github.com/alibabacloud-go/darabonba-encode-util v0.0.2 // indirect
+	github.com/alibabacloud-go/darabonba-map v0.0.2 // indirect
+	github.com/alibabacloud-go/darabonba-signature-util v0.0.7 // indirect
+	github.com/alibabacloud-go/darabonba-string v1.0.2 // indirect
+	github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 // indirect
+	github.com/alibabacloud-go/openapi-util v0.1.0 // indirect
+	github.com/alibabacloud-go/tea-utils v1.3.1 // indirect
+	github.com/alibabacloud-go/tea-utils/v2 v2.0.4 // indirect
+	github.com/alibabacloud-go/tea-xml v1.1.3 // indirect
+	github.com/aliyun/credentials-go v1.3.1 // indirect
+	github.com/clbanning/mxj/v2 v2.5.5 // indirect
+	github.com/tjfoc/gmsm v1.4.1 // indirect
+)
 
 replace (
 	github.com/VictoriaMetrics/VictoriaMetrics => github.com/iLogtail/VictoriaMetrics v1.83.4-ilogtail

helper/envconfig/aliyun_ecs_meta_data.go

@@ -89,7 +89,7 @@ func getToken() (result []byte, err error) {
 	role := roles[0]
 
 	var respGet *http.Response
-	logger.Debug(context.Background(), "get token request", aliyunECSRamURL+role)
+	logger.Info(context.Background(), "get token request", aliyunECSRamURL+role)
 	respGet, err = client.Get(aliyunECSRamURL + role)
 	if err != nil {
 		logger.Warning(context.Background(), "UPDATE_STS_ALARM", "get token error", err, "role", role)

helper/envconfig/aliyun_log_config_define.go

@@ -22,7 +22,9 @@ import (
 	"strings"
 	"time"
 
-	aliyunlog "github.com/aliyun/aliyun-log-go-sdk"
+	"github.com/alibabacloud-go/tea/tea"
+
+	aliyunlog "github.com/alibabacloud-go/sls-20201230/v5/client"
 	"github.com/gogo/protobuf/proto"
 
 	"github.com/alibaba/ilogtail/pkg/flags"
@@ -93,29 +95,29 @@ import (
 
 // AliyunLogConfigSpec logtail config struct for wrapper
 type AliyunLogConfigSpec struct {
-	Project                string                `json:"project"`
-	Logstore               string                `json:"logstore"`
-	ShardCount             *int32                `json:"shardCount"`
-	LifeCycle              *int32                `json:"lifeCycle"`
-	MachineGroups          []string              `json:"machineGroups"`
-	LogtailConfig          AliyunLogConfigDetail `json:"logtailConfig"`
-	ContainerName          string                `json:"containerName"`
-	HashCode               []byte                `json:"hashCode"`
-	ErrorCount             int                   `json:"errorCount"`
-	NextTryTime            int64                 `json:"nextTryTime"`
-	SimpleConfig           bool                  `json:"simpleConfig"`
-	LastFetchTime          int64                 `json:"lastFetchTime"`
-	ProductCode            string                `json:"productCode"`
-	ProductLang            string                `json:"productLang"`
-	LogstoreMode           string                `json:"logstoreMode"`
-	LogstoreHotTTL         *int32                `json:"logstoreHotTTL"`
-	LogstoreTelemetryType  string                `json:"logstoreTelemetryType"`
-	LogstoreAppendMeta     bool                  `json:"logstoreAppendMeta"`
-	LogstoreMaxSplitShard  *int32                `json:"logstoreMaxSplitShard"`
-	LogstoreEnableTracking bool                  `json:"logstoreEnableTracking"`
-	LogstoreAutoSplit      bool                  `json:"logstoreAutoSplit"`
-	LogstoreEncryptConf    aliyunlog.EncryptConf `json:"logstoreEncryptConf"`
-	ConfigTags             map[string]string     `json:"configTags"`
+	Project                string                          `json:"project"`
+	Logstore               string                          `json:"logstore"`
+	ShardCount             *int32                          `json:"shardCount"`
+	LifeCycle              *int32                          `json:"lifeCycle"`
+	MachineGroups          []string                        `json:"machineGroups"`
+	LogtailConfig          aliyunlog.LogtailPipelineConfig `json:"logtailConfig"`
+	ContainerName          string                          `json:"containerName"`
+	HashCode               []byte                          `json:"hashCode"`
+	ErrorCount             int                             `json:"errorCount"`
+	NextTryTime            int64                           `json:"nextTryTime"`
+	SimpleConfig           bool                            `json:"simpleConfig"`
+	LastFetchTime          int64                           `json:"lastFetchTime"`
+	ProductCode            string                          `json:"productCode"`
+	ProductLang            string                          `json:"productLang"`
+	LogstoreMode           string                          `json:"logstoreMode"`
+	LogstoreHotTTL         *int32                          `json:"logstoreHotTTL"`
+	LogstoreTelemetryType  string                          `json:"logstoreTelemetryType"`
+	LogstoreAppendMeta     bool                            `json:"logstoreAppendMeta"`
+	LogstoreMaxSplitShard  *int32                          `json:"logstoreMaxSplitShard"`
+	LogstoreEnableTracking bool                            `json:"logstoreEnableTracking"`
+	LogstoreAutoSplit      bool                            `json:"logstoreAutoSplit"`
+	LogstoreEncryptConf    aliyunlog.EncryptConf           `json:"logstoreEncryptConf"`
+	ConfigTags             map[string]string               `json:"configTags"`
 }
 
 const (
@@ -131,13 +133,6 @@ const (
 	SlsMachinegroupDeployModeDeamonset string = "deamonset"
 )
 
-// AliyunLogConfigDetail logtail config detail
-type AliyunLogConfigDetail struct {
-	ConfigName    string                 `json:"configName"`
-	InputType     string                 `json:"inputType"`
-	LogtailConfig map[string]interface{} `json:"inputDetail"`
-}
-
 // Hash generate sha256 hash
 func (alcs *AliyunLogConfigSpec) hash(hashValue string) {
 	h := sha256.New()
@@ -147,16 +142,7 @@ func (alcs *AliyunLogConfigSpec) hash(hashValue string) {
 
 // Key return the unique key
 func (alcs *AliyunLogConfigSpec) Key() string {
-	return alcs.Project + "@" + alcs.LogtailConfig.ConfigName + "@" + alcs.ContainerName
-}
-
-// InitFromDockerInfo init AliyunLogConfigDetail from docker info with specific config name
-func (alcs *AliyunLogConfigDetail) InitFromDockerInfo(dockerInfo *helper.DockerInfoDetail,
-	config string,
-	configKeys []string,
-	configValues []string) error {
-
-	return nil
+	return alcs.Project + "@" + tea.StringValue(alcs.LogtailConfig.ConfigName) + "@" + alcs.ContainerName
 }
 
 // all log config spec, must been cleared before next process
@@ -183,8 +169,9 @@ func isDockerStdout(configType string) bool {
 
 // nolint: unparam
 func initDockerStdoutConfig(dockerInfo *helper.DockerInfoDetail, config *AliyunLogConfigSpec, configType string) {
-	config.LogtailConfig.InputType = "plugin"
+	config.LogtailConfig.Inputs = make([]map[string]interface{}, 0, 1)
 	stdoutDetail := make(map[string]interface{})
+	stdoutDetail["Type"] = "service_docker_stdout"
 	switch configType {
 	case "stdout":
 		stdoutDetail["Stdout"] = true
@@ -204,54 +191,62 @@ func initDockerStdoutConfig(dockerInfo *helper.DockerInfoDetail, config *AliyunL
 	// 	}
 	// }
 	stdoutDetail["IncludeEnv"] = map[string]string{
-		*flags.LogConfigPrefix + config.LogtailConfig.ConfigName: configType,
-	}
-	stdoutConfig := map[string]interface{}{
-		"type":   "service_docker_stdout",
-		"detail": stdoutDetail,
-	}
-	config.LogtailConfig.LogtailConfig = map[string]interface{}{
-		"plugin": map[string]interface{}{
-			"inputs": []interface{}{stdoutConfig},
-			"global": map[string]interface{}{
-				"AlwaysOnline": true,
-			},
-		},
+		*flags.LogConfigPrefix + tea.StringValue(config.LogtailConfig.ConfigName): configType,
 	}
+	config.LogtailConfig.Inputs = append(config.LogtailConfig.Inputs, stdoutDetail)
+	config.LogtailConfig.Global = map[string]interface{}{"AlwaysOnline": true}
 }
 
 const invalidFilePattern = "invalid_file_pattern"
 
 // nolint: unparam
 func initFileConfig(k8sInfo *helper.K8SInfo, config *AliyunLogConfigSpec, filePath string, jsonFlag, dockerFile bool) {
-	config.LogtailConfig.InputType = "file"
+	config.LogtailConfig.Inputs = make([]map[string]interface{}, 0, 1)
+
+	// 判断有没有 /**/
+	maxDirSearchDepth := 0
+	if strings.Contains(filePath, "/**/") {
+		maxDirSearchDepth = 10
+	}
+
 	logPath, filePattern, err := splitLogPathAndFilePattern(filePath)
+	var input map[string]interface{}
 	if err != nil {
 		logger.Error(context.Background(), "INVALID_DOCKER_ENV_CONFIG_ALARM", "invalid file config, you must input full file path", filePath, err)
 	}
 
 	if !jsonFlag {
-		config.LogtailConfig.LogtailConfig = map[string]interface{}{
-			"logType":     "common_reg_log",
-			"logPath":     logPath,
-			"filePattern": filePattern,
-			"dockerFile":  dockerFile,
-			"dockerIncludeEnv": map[string]string{
-				*flags.LogConfigPrefix + config.LogtailConfig.ConfigName: filePath,
+		input = map[string]interface{}{
+			"Type":                     "input_file",
+			"FilePaths":                []string{logPath + filePattern},
+			"EnableContainerDiscovery": dockerFile,
+			"ContainerFilters": map[string]map[string]interface{}{
+				"IncludeEnv": {
+					*flags.LogConfigPrefix + tea.StringValue(config.LogtailConfig.ConfigName): filePath,
+				},
 			},
+			"MaxDirSearchDepth": maxDirSearchDepth,
 		}
 	} else {
-		config.LogtailConfig.LogtailConfig = map[string]interface{}{
-			"logType":     "json_log",
-			"logPath":     logPath,
-			"filePattern": filePattern,
-			"dockerFile":  dockerFile,
-			"dockerIncludeEnv": map[string]string{
-				*flags.LogConfigPrefix + config.LogtailConfig.ConfigName: filePath,
+		input = map[string]interface{}{
+			"Type":                     "input_file",
+			"FilePaths":                []string{logPath + filePattern},
+			"EnableContainerDiscovery": dockerFile,
+			"ContainerFilters": map[string]map[string]interface{}{
+				"IncludeEnv": {
+					*flags.LogConfigPrefix + tea.StringValue(config.LogtailConfig.ConfigName): filePath,
+				},
+			},
+			"MaxDirSearchDepth": maxDirSearchDepth,
+		}
+		config.LogtailConfig.Processors = []map[string]interface{}{
+			{
+				"SourceKey": "content",
+				"Type":      "processor_parse_json_native",
 			},
 		}
 	}
-
+	config.LogtailConfig.Inputs = append(config.LogtailConfig.Inputs, input)
 	//  remove container name limit, this will not work when pod with a sidecar/init container which has same env
 	// if len(k8sInfo.Namespace) > 0 {
 	// 	config.LogtailConfig.LogtailConfig["dockerIncludeLabel"] = map[string]string{
@@ -261,6 +256,7 @@ func initFileConfig(k8sInfo *helper.K8SInfo, config *AliyunLogConfigSpec, filePa
 }
 
 func isJSONFile(envConfigInfo *helper.EnvConfigInfo, config *AliyunLogConfigSpec) bool {
+	// aliyun_logs_****_jsonfile=true or aliyun_logs_****_jsonfile=TRUE
 	if jsonfile, ok := envConfigInfo.ConfigItemMap["jsonfile"]; ok && (jsonfile == "true" || jsonfile == "TRUE") {
 		return true
 	}
@@ -285,7 +281,7 @@ func makeLogConfigSpec(dockerInfo *helper.DockerInfoDetail, envConfigInfo *helpe
 	config.LastFetchTime = time.Now().Unix()
 	// save all config info, and keep sequence
 	var totalConfig string
-	config.LogtailConfig.ConfigName = envConfigInfo.ConfigName
+	config.LogtailConfig.ConfigName = tea.String(envConfigInfo.ConfigName)
 	// @note add image name to prevent key conflict
 	if len(dockerInfo.K8SInfo.ContainerName) > 0 {
 		config.ContainerName = dockerInfo.K8SInfo.ContainerName
@@ -414,13 +410,12 @@ func makeLogConfigSpec(dockerInfo *helper.DockerInfoDetail, envConfigInfo *helpe
 	// config
 	// makesure exist
 	configType := envConfigInfo.ConfigItemMap[""]
-	config.LogtailConfig.LogtailConfig = make(map[string]interface{})
+
 	if configDetail, ok := envConfigInfo.ConfigItemMap["detail"]; ok {
 		totalConfig += configDetail
-		if err := json.Unmarshal([]byte(configDetail), &config.LogtailConfig.LogtailConfig); err != nil {
-			logger.Error(context.Background(), "INVALID_ENV_CONFIG_DETAIL", "unmarshal error", err, "detail", configDetail)
-		}
-		config.LogtailConfig.InputType = configType
+		// if err := json.Unmarshal([]byte(configDetail), &config.LogtailConfig); err != nil {
+		// 	logger.Error(context.Background(), "INVALID_ENV_CONFIG_DETAIL", "unmarshal error", err, "detail", configDetail)
+		// }
 		config.SimpleConfig = false
 	} else {
 		totalConfig += configType

helper/envconfig/aliyun_log_config_define_others.go

@@ -22,7 +22,7 @@ import (
 	"strings"
 )
 
-const invalidLogPath = "/invalid_log_path"
+const invalidLogPath = "/invalid_log_path/"
 
 func splitLogPathAndFilePattern(filePath string) (logPath string, filePattern string, err error) {
 	lastSeperatorPos := strings.LastIndexByte(filePath, '/')