From 0a2a9289520d2d04343034a24e327943b4e2e52c Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 29 Mar 2024 14:43:26 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E5=AE=B9=E5=99=A8=E5=85=83=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E9=A2=84=E8=A7=88=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/input/InputFile.cpp | 2 +- plugins/input/docker/logmeta/metric_docker_file.go | 8 +++++++- plugins/input/docker/stdout/input_docker_stdout.go | 8 +++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/core/input/InputFile.cpp b/core/input/InputFile.cpp index f2371654ad..7f1097a46c 100644 --- a/core/input/InputFile.cpp +++ b/core/input/InputFile.cpp @@ -206,7 +206,7 @@ void InputFile::GenerateContainerMetaFetchingGoPipeline(Json::Value& res) const ConvertMapToJsonObj("ExternalK8sLabelTag", mContainerDiscovery.mExternalK8sLabelTag); ConvertMapToJsonObj("ExternalEnvTag", mContainerDiscovery.mExternalEnvTag); if (mContainerDiscovery.mCollectingContainersMeta) { - detail["CollectingContainersMeta"] = Json::Value(true); + detail["CollectContainersFlag"] = Json::Value(true); } plugin["type"] = Json::Value("metric_docker_file"); plugin["detail"] = detail; diff --git a/plugins/input/docker/logmeta/metric_docker_file.go b/plugins/input/docker/logmeta/metric_docker_file.go index 48065e9a5b..72b0c3eeab 100644 --- a/plugins/input/docker/logmeta/metric_docker_file.go +++ b/plugins/input/docker/logmeta/metric_docker_file.go @@ -344,11 +344,17 @@ func (idf *InputDockerFile) Collect(collector pipeline.Collector) error { } } if idf.CollectContainersFlag { + var configResult *helper.ContainerConfigResult + configName := idf.context.GetConfigName() + lastSlashIndex := strings.LastIndex(configName, "/") + if lastSlashIndex != -1 { + configName = configName[:lastSlashIndex] + } configResult := &helper.ContainerConfigResult{ DataType: "container_config_result", Project: idf.context.GetProject(), Logstore: idf.context.GetLogstore(), - ConfigName: idf.context.GetConfigName(), + ConfigName: configName, SourceAddress: fmt.Sprintf("%s/**/%s", idf.LogPath, idf.FilePattern), PathExistInputContainerIDs: helper.GetStringFromList(havingPathkeys), PathNotExistInputContainerIDs: helper.GetStringFromList(nothavingPathkeys), diff --git a/plugins/input/docker/stdout/input_docker_stdout.go b/plugins/input/docker/stdout/input_docker_stdout.go index cda7138a6c..04070d9bb8 100644 --- a/plugins/input/docker/stdout/input_docker_stdout.go +++ b/plugins/input/docker/stdout/input_docker_stdout.go @@ -17,6 +17,7 @@ package stdout import ( "fmt" "regexp" + "strings" "sync" "time" @@ -278,11 +279,16 @@ func (sds *ServiceDockerStdout) FlushAll(c pipeline.Collector, firstStart bool) keys = append(keys, helper.GetShortID(k)) } } + configName := sds.context.GetConfigName() + lastSlashIndex := strings.LastIndex(configName, "/") + if lastSlashIndex != -1 { + configName = configName[:lastSlashIndex] + } configResult := &helper.ContainerConfigResult{ DataType: "container_config_result", Project: sds.context.GetProject(), Logstore: sds.context.GetLogstore(), - ConfigName: sds.context.GetConfigName(), + ConfigName: configName, PathExistInputContainerIDs: helper.GetStringFromList(keys), SourceAddress: "stdout", InputType: input.ServiceDockerStdoutPluginName, From 2d9a7ef3c615f9386e8eb1b50da6244627ad9692 Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Mon, 1 Apr 2024 06:54:01 +0000 Subject: [PATCH 2/6] add GetBaseConfigName --- pkg/util/util.go | 8 ++++++++ plugins/input/docker/logmeta/metric_docker_file.go | 8 +------- plugins/input/docker/stdout/input_docker_stdout.go | 8 +------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index 05e52e4564..22933a8f18 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -399,3 +399,11 @@ func UniqueStrings(strSlices ...[]string) []string { } return result } + +func GetBaseConfigName(configName string) string { + lastSlashIndex := strings.LastIndex(configName, "/") + if lastSlashIndex != -1 { + configName = configName[:lastSlashIndex] + } + return configName +} diff --git a/plugins/input/docker/logmeta/metric_docker_file.go b/plugins/input/docker/logmeta/metric_docker_file.go index 72b0c3eeab..d910c96f4c 100644 --- a/plugins/input/docker/logmeta/metric_docker_file.go +++ b/plugins/input/docker/logmeta/metric_docker_file.go @@ -344,17 +344,11 @@ func (idf *InputDockerFile) Collect(collector pipeline.Collector) error { } } if idf.CollectContainersFlag { - var configResult *helper.ContainerConfigResult - configName := idf.context.GetConfigName() - lastSlashIndex := strings.LastIndex(configName, "/") - if lastSlashIndex != -1 { - configName = configName[:lastSlashIndex] - } configResult := &helper.ContainerConfigResult{ DataType: "container_config_result", Project: idf.context.GetProject(), Logstore: idf.context.GetLogstore(), - ConfigName: configName, + ConfigName: util.GetBaseConfigName(idf.context.GetConfigName()), SourceAddress: fmt.Sprintf("%s/**/%s", idf.LogPath, idf.FilePattern), PathExistInputContainerIDs: helper.GetStringFromList(havingPathkeys), PathNotExistInputContainerIDs: helper.GetStringFromList(nothavingPathkeys), diff --git a/plugins/input/docker/stdout/input_docker_stdout.go b/plugins/input/docker/stdout/input_docker_stdout.go index 04070d9bb8..c45b2947ec 100644 --- a/plugins/input/docker/stdout/input_docker_stdout.go +++ b/plugins/input/docker/stdout/input_docker_stdout.go @@ -17,7 +17,6 @@ package stdout import ( "fmt" "regexp" - "strings" "sync" "time" @@ -279,16 +278,11 @@ func (sds *ServiceDockerStdout) FlushAll(c pipeline.Collector, firstStart bool) keys = append(keys, helper.GetShortID(k)) } } - configName := sds.context.GetConfigName() - lastSlashIndex := strings.LastIndex(configName, "/") - if lastSlashIndex != -1 { - configName = configName[:lastSlashIndex] - } configResult := &helper.ContainerConfigResult{ DataType: "container_config_result", Project: sds.context.GetProject(), Logstore: sds.context.GetLogstore(), - ConfigName: configName, + ConfigName: util.GetBaseConfigName(sds.context.GetConfigName()), PathExistInputContainerIDs: helper.GetStringFromList(keys), SourceAddress: "stdout", InputType: input.ServiceDockerStdoutPluginName, From a7567f974fc60e76ae1bb072b5f6dfcb74b6d2cb Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Mon, 1 Apr 2024 11:10:13 +0000 Subject: [PATCH 3/6] CollectingContainersMeta --- core/input/InputFile.cpp | 2 +- pluginmanager/logstore_config.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/input/InputFile.cpp b/core/input/InputFile.cpp index 7f1097a46c..f2371654ad 100644 --- a/core/input/InputFile.cpp +++ b/core/input/InputFile.cpp @@ -206,7 +206,7 @@ void InputFile::GenerateContainerMetaFetchingGoPipeline(Json::Value& res) const ConvertMapToJsonObj("ExternalK8sLabelTag", mContainerDiscovery.mExternalK8sLabelTag); ConvertMapToJsonObj("ExternalEnvTag", mContainerDiscovery.mExternalEnvTag); if (mContainerDiscovery.mCollectingContainersMeta) { - detail["CollectContainersFlag"] = Json::Value(true); + detail["CollectingContainersMeta"] = Json::Value(true); } plugin["type"] = Json::Value("metric_docker_file"); plugin["detail"] = detail; diff --git a/pluginmanager/logstore_config.go b/pluginmanager/logstore_config.go index 583dc9a0df..93cf13765a 100644 --- a/pluginmanager/logstore_config.go +++ b/pluginmanager/logstore_config.go @@ -488,6 +488,12 @@ func createLogstoreConfig(project string, logstore string, configName string, lo continue } logstoreC.CollectContainersFlag = collectContainersFlag + } else if strings.Contains(lowerKey, "collectingcontainersmeta") { + collectContainersFlag, valid := value.(bool) + if !valid { + continue + } + logstoreC.CollectContainersFlag = collectContainersFlag } } } From a3e7f5d11cc1ca3e0e99a89d026a42ee2fc2e6fa Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Mon, 1 Apr 2024 11:36:24 +0000 Subject: [PATCH 4/6] CollectingContainersMeta --- pluginmanager/container_config_manager.go | 6 +++--- pluginmanager/logstore_config.go | 12 ++++++------ plugins/input/docker/logmeta/metric_docker_file.go | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pluginmanager/container_config_manager.go b/pluginmanager/container_config_manager.go index 946e5f582c..2b71dc462e 100644 --- a/pluginmanager/container_config_manager.go +++ b/pluginmanager/container_config_manager.go @@ -129,7 +129,7 @@ func refreshEnvAndLabel() { containerLabelSet = make(map[string]struct{}) k8sLabelSet = make(map[string]struct{}) for _, logstoreConfig := range LogtailConfig { - if logstoreConfig.CollectContainersFlag { + if logstoreConfig.CollectingContainersMeta { for key := range logstoreConfig.EnvSet { envSet[key] = struct{}{} } @@ -150,7 +150,7 @@ func compareEnvAndLabel() (diffEnvSet, diffContainerLabelSet, diffK8sLabelSet ma diffContainerLabelSet = make(map[string]struct{}) diffK8sLabelSet = make(map[string]struct{}) for _, logstoreConfig := range LogtailConfig { - if logstoreConfig.CollectContainersFlag { + if logstoreConfig.CollectingContainersMeta { for key := range logstoreConfig.EnvSet { if _, ok := envSet[key]; !ok { envSet[key] = struct{}{} @@ -268,7 +268,7 @@ func compareEnvAndLabelAndRecordContainer() []*helper.ContainerDetail { func isCollectContainers() bool { for _, logstoreConfig := range LogtailConfig { - if logstoreConfig.CollectContainersFlag { + if logstoreConfig.CollectingContainersMeta { return true } } diff --git a/pluginmanager/logstore_config.go b/pluginmanager/logstore_config.go index 93cf13765a..36a2d777d4 100644 --- a/pluginmanager/logstore_config.go +++ b/pluginmanager/logstore_config.go @@ -113,10 +113,10 @@ type LogstoreConfig struct { // flushWaitSema sync.WaitGroup pauseOrResumeWg sync.WaitGroup - K8sLabelSet map[string]struct{} - ContainerLabelSet map[string]struct{} - EnvSet map[string]struct{} - CollectContainersFlag bool + K8sLabelSet map[string]struct{} + ContainerLabelSet map[string]struct{} + EnvSet map[string]struct{} + CollectingContainersMeta bool } func (p *LogstoreStatistics) Init(context pipeline.Context) { @@ -487,13 +487,13 @@ func createLogstoreConfig(project string, logstore string, configName string, lo if !valid { continue } - logstoreC.CollectContainersFlag = collectContainersFlag + logstoreC.CollectingContainersMeta = collectContainersFlag } else if strings.Contains(lowerKey, "collectingcontainersmeta") { collectContainersFlag, valid := value.(bool) if !valid { continue } - logstoreC.CollectContainersFlag = collectContainersFlag + logstoreC.CollectingContainersMeta = collectContainersFlag } } } diff --git a/plugins/input/docker/logmeta/metric_docker_file.go b/plugins/input/docker/logmeta/metric_docker_file.go index d910c96f4c..4180ca05cf 100644 --- a/plugins/input/docker/logmeta/metric_docker_file.go +++ b/plugins/input/docker/logmeta/metric_docker_file.go @@ -87,10 +87,10 @@ type InputDockerFile struct { lastUpdateTime int64 // Last return of GetAllAcceptedInfoV2 - fullList map[string]bool - matchList map[string]*helper.DockerInfoDetail - CollectContainersFlag bool - firstStart bool + fullList map[string]bool + matchList map[string]*helper.DockerInfoDetail + CollectingContainersMeta bool + firstStart bool } func formatPath(path string) string { @@ -343,7 +343,7 @@ func (idf *InputDockerFile) Collect(collector pipeline.Collector) error { logger.Warning(idf.context.GetRuntimeContext(), "DOCKER_FILE_MATCH_ALARM", "unknow error", "can't find path from this container", "path", idf.LogPath, "container", info.ContainerInfo.Name) } } - if idf.CollectContainersFlag { + if idf.CollectingContainersMeta { configResult := &helper.ContainerConfigResult{ DataType: "container_config_result", Project: idf.context.GetProject(), From 0158551d3d7e7cc318b5689b90b90398c0fccf06 Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Mon, 1 Apr 2024 11:37:41 +0000 Subject: [PATCH 5/6] fix --- pluginmanager/logstore_config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pluginmanager/logstore_config.go b/pluginmanager/logstore_config.go index 36a2d777d4..38d036c73c 100644 --- a/pluginmanager/logstore_config.go +++ b/pluginmanager/logstore_config.go @@ -489,11 +489,11 @@ func createLogstoreConfig(project string, logstore string, configName string, lo } logstoreC.CollectingContainersMeta = collectContainersFlag } else if strings.Contains(lowerKey, "collectingcontainersmeta") { - collectContainersFlag, valid := value.(bool) + collectingContainersMeta, valid := value.(bool) if !valid { continue } - logstoreC.CollectingContainersMeta = collectContainersFlag + logstoreC.CollectingContainersMeta = collectingContainersMeta } } } From 4a81f0deeaf3e5fe99ab34bbb8023dff580eebe0 Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Tue, 2 Apr 2024 16:48:04 +0800 Subject: [PATCH 6/6] delete GetBaseConfigName --- pkg/util/util.go | 8 -------- plugins/input/docker/logmeta/metric_docker_file.go | 2 +- plugins/input/docker/stdout/input_docker_stdout.go | 2 +- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index 22933a8f18..05e52e4564 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -399,11 +399,3 @@ func UniqueStrings(strSlices ...[]string) []string { } return result } - -func GetBaseConfigName(configName string) string { - lastSlashIndex := strings.LastIndex(configName, "/") - if lastSlashIndex != -1 { - configName = configName[:lastSlashIndex] - } - return configName -} diff --git a/plugins/input/docker/logmeta/metric_docker_file.go b/plugins/input/docker/logmeta/metric_docker_file.go index 4180ca05cf..ef6e2e4b6f 100644 --- a/plugins/input/docker/logmeta/metric_docker_file.go +++ b/plugins/input/docker/logmeta/metric_docker_file.go @@ -348,7 +348,7 @@ func (idf *InputDockerFile) Collect(collector pipeline.Collector) error { DataType: "container_config_result", Project: idf.context.GetProject(), Logstore: idf.context.GetLogstore(), - ConfigName: util.GetBaseConfigName(idf.context.GetConfigName()), + ConfigName: idf.context.GetConfigName(), SourceAddress: fmt.Sprintf("%s/**/%s", idf.LogPath, idf.FilePattern), PathExistInputContainerIDs: helper.GetStringFromList(havingPathkeys), PathNotExistInputContainerIDs: helper.GetStringFromList(nothavingPathkeys), diff --git a/plugins/input/docker/stdout/input_docker_stdout.go b/plugins/input/docker/stdout/input_docker_stdout.go index c45b2947ec..cda7138a6c 100644 --- a/plugins/input/docker/stdout/input_docker_stdout.go +++ b/plugins/input/docker/stdout/input_docker_stdout.go @@ -282,7 +282,7 @@ func (sds *ServiceDockerStdout) FlushAll(c pipeline.Collector, firstStart bool) DataType: "container_config_result", Project: sds.context.GetProject(), Logstore: sds.context.GetLogstore(), - ConfigName: util.GetBaseConfigName(sds.context.GetConfigName()), + ConfigName: sds.context.GetConfigName(), PathExistInputContainerIDs: helper.GetStringFromList(keys), SourceAddress: "stdout", InputType: input.ServiceDockerStdoutPluginName,