diff --git a/logstash-core/src/main/java/org/logstash/log/PipelineRoutingAppender.java b/logstash-core/src/main/java/org/logstash/log/PipelineRoutingAppender.java index 4f8ffccaf9..ba54ed60a8 100644 --- a/logstash-core/src/main/java/org/logstash/log/PipelineRoutingAppender.java +++ b/logstash-core/src/main/java/org/logstash/log/PipelineRoutingAppender.java @@ -2,6 +2,7 @@ import org.apache.logging.log4j.core.Appender; import org.apache.logging.log4j.core.Core; +import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.appender.AbstractAppender; import org.apache.logging.log4j.core.config.AppenderControl; @@ -107,8 +108,11 @@ public void append(LogEvent event) { private AppenderControl getControl(LogEvent event) { String key = event.getContextData().getValue("pipeline.id"); if (key == null) { - error("Unable to find the pipeline.id in event's context data"); - key = "sink"; + LOGGER.debug("Unable to find the pipeline.id in event's context data in routing appender, skip it"); + // this prevent to create an appender when log events are not fish-tagged with pipeline.id, + // avoid to create log file like "pipeline_${ctx:pipeline.id}.log" which contains duplicated + // logs from the logstash-* files + return null; } AppenderControl appenderControl = createdAppenders.get(key); diff --git a/logstash-core/src/main/java/org/logstash/log/PipelineRoutingFilter.java b/logstash-core/src/main/java/org/logstash/log/PipelineRoutingFilter.java index 544065fe5b..a4e59a27ce 100644 --- a/logstash-core/src/main/java/org/logstash/log/PipelineRoutingFilter.java +++ b/logstash-core/src/main/java/org/logstash/log/PipelineRoutingFilter.java @@ -13,7 +13,7 @@ @Plugin(name = "PipelineRoutingFilter", category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE, printObject = true) public final class PipelineRoutingFilter extends AbstractFilter { - private boolean isSeparateLogs; + private final boolean isSeparateLogs; /** * Factory method to instantiate the filter diff --git a/qa/integration/build.gradle b/qa/integration/build.gradle index 2eb03ca4ce..fc94f05be9 100644 --- a/qa/integration/build.gradle +++ b/qa/integration/build.gradle @@ -51,6 +51,8 @@ tasks.register("copyProductionLog4jConfiguration", Copy) { 'appender.rolling.policies.size.size = 1KB') .replace('appender.rolling.filePattern = ${sys:ls.logs}/logstash-plain-%d{yyyy-MM-dd}-%i.log.gz', 'appender.rolling.filePattern = ${sys:ls.logs}/logstash-plain-%d{yyyy-MM-dd}.log') + .replace('appender.routing.pipeline.policy.size = 100MB', + 'appender.routing.pipeline.policy.size = 1KB') } } diff --git a/qa/integration/specs/pipeline_log_spec.rb b/qa/integration/specs/pipeline_log_spec.rb index 1e3bf575bc..b91e4a03f3 100644 --- a/qa/integration/specs/pipeline_log_spec.rb +++ b/qa/integration/specs/pipeline_log_spec.rb @@ -142,6 +142,29 @@ end end + it "rollover of pipeline log file when pipeline.separate_logs is enabled shouldn't create spurious file " do + pipeline_name = "custom_pipeline" + settings = { + "path.logs" => temp_dir, + "pipeline.id" => pipeline_name, + "pipeline.separate_logs" => true + } + FileUtils.mkdir_p(File.join(temp_dir, "data")) + data = File.join(temp_dir, "data") + settings = settings.merge({ "path.data" => data }) + IO.write(File.join(temp_dir, "logstash.yml"), YAML.dump(settings)) + + log_definition = File.read('fixtures/logs_rollover/log4j2.properties') + expect(log_definition).to match(/appender\.routing\.pipeline\.policy\.size\s*=\s*1KB/) + FileUtils.cp("fixtures/logs_rollover/log4j2.properties", temp_dir) + + @ls.spawn_logstash("--path.settings", temp_dir, "-w", "1" , "-e", config) + wait_logstash_process_terminate(@ls) + + pipeline_logs = Dir.glob("pipeline*.log", base: temp_dir) + expect(pipeline_logs).not_to include("pipeline_${ctx:pipeline.id}.log") + end + it "should not create separate pipelines log files if not enabled" do pipeline_name = "custom_pipeline" settings = {