From 6978442abc55ada48be4698c05fa6e3178e69b14 Mon Sep 17 00:00:00 2001 From: majimenez-stratio Date: Sun, 31 Oct 2021 02:14:45 +0100 Subject: [PATCH 1/7] Fix stat_watcher wrong path Signed-off-by: majimenez-stratio --- lib/fluent/plugin/in_tail.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb index 0ee8bab77f..3cdbd4faa9 100644 --- a/lib/fluent/plugin/in_tail.rb +++ b/lib/fluent/plugin/in_tail.rb @@ -370,7 +370,7 @@ def refresh_watchers target_paths_hash = expand_paths existence_paths_hash = existence_path - log.debug { "tailing paths: target = #{target_paths.join(",")} | existing = #{existence_paths.join(",")}" } + log.debug { "tailing paths: target = #{target_paths_hash.keys.join(",")} | existing = #{existence_paths_hash.keys.join(",")}" } unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)} added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)} @@ -389,7 +389,7 @@ def setup_watcher(target_info, pe) end if @enable_stat_watcher - tt = StatWatcher.new(path, log) { tw.on_notify } + tt = StatWatcher.new(target_info.path, log) { tw.on_notify } tw.register_watcher(tt) end From 1dae546bc8a98a51b36db18d61e7ece47bd8ff83 Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Mon, 1 Nov 2021 21:35:59 +0900 Subject: [PATCH 2/7] test_in_tail: Fix an inappropriate config name CONFIG_ENABLE_WATCH_TIMER -> CONFIG_DISABLE_WATCH_TIMER The tests that use above config are intended to disable watch timer. Signed-off-by: Takuro Ashie --- test/plugin/test_in_tail.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb index 89296da69a..5830773821 100644 --- a/test/plugin/test_in_tail.rb +++ b/test/plugin/test_in_tail.rb @@ -99,7 +99,7 @@ def create_target_info(path) }) COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" }) CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true }) - CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false }) + CONFIG_DISABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false }) CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false }) CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true }) COMMON_FOLLOW_INODE_CONFIG = config_element("ROOT", "", { @@ -199,7 +199,7 @@ def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true) sub_test_case "log throttling per file" do test "w/o watcher timer is invalid" do - conf = CONFIG_ENABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"}) + conf = CONFIG_DISABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"}) assert_raise(Fluent::ConfigError) do create_driver(conf) end @@ -215,7 +215,7 @@ def create_driver(conf = SINGLE_LINE_CONFIG, use_common_conf = true) test "both enable_watch_timer and enable_stat_watcher are false" do assert_raise(Fluent::ConfigError) do - create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG) + create_driver(CONFIG_DISABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG) end end @@ -570,9 +570,9 @@ def test_emit_with_read_from_head(data) assert_equal({"message" => "test4"}, events[3][2]) end - data(flat: CONFIG_ENABLE_WATCH_TIMER + SINGLE_LINE_CONFIG, - parse: CONFIG_ENABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG) - def test_emit_with_enable_watch_timer(data) + data(flat: CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG, + parse: CONFIG_DISABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG) + def test_emit_without_watch_timer(data) config = data File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test1" From 494055e3c23250f4f09ce28bb0cbde658f37f17d Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Mon, 1 Nov 2021 22:13:42 +0900 Subject: [PATCH 3/7] test_in_tail: Add test_watch_wildcard_path_without_watch_timer https://github.com/fluent/fluentd/pull/3541#discussion_r740197711 Signed-off-by: Takuro Ashie --- test/plugin/test_in_tail.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb index 5830773821..c312be54d1 100644 --- a/test/plugin/test_in_tail.rb +++ b/test/plugin/test_in_tail.rb @@ -596,6 +596,35 @@ def test_emit_without_watch_timer(data) assert_equal({"message" => "test4"}, events[1][2]) end + # https://github.com/fluent/fluentd/pull/3541#discussion_r740197711 + def test_watch_wildcard_path_without_watch_timer + config = config_element("ROOT", "", { + "path" => "#{TMP_DIR}/tail*.txt", + "tag" => "t1", + }) + config = config + CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG + File.open("#{TMP_DIR}/tail.txt", "wb") {|f| + f.puts "test1" + f.puts "test2" + } + + d = create_driver(config, false) + + d.run(expect_emits: 1) do + File.open("#{TMP_DIR}/tail.txt", "ab") {|f| + f.puts "test3" + f.puts "test4" + } + end + + assert_equal( + [ + {"message" => "test3"}, + {"message" => "test4"}, + ], + d.events.collect { |event| event[2] }) + end + data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG, parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG) def test_emit_with_disable_stat_watcher(data) From e908a5dfef5807f05effc979677d9c17a1cc69f0 Mon Sep 17 00:00:00 2001 From: majimenez-stratio Date: Mon, 1 Nov 2021 16:37:05 +0100 Subject: [PATCH 4/7] Fix log when follow_inodes = true Signed-off-by: majimenez-stratio --- lib/fluent/plugin/in_tail.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb index 3cdbd4faa9..efeed94af2 100644 --- a/lib/fluent/plugin/in_tail.rb +++ b/lib/fluent/plugin/in_tail.rb @@ -369,8 +369,10 @@ def existence_path def refresh_watchers target_paths_hash = expand_paths existence_paths_hash = existence_path - - log.debug { "tailing paths: target = #{target_paths_hash.keys.join(",")} | existing = #{existence_paths_hash.keys.join(",")}" } + + target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",") + existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",") + log.debug { "tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}" } unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)} added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)} From 8c03b7150cbef4220b881ef5c433fca236dffde2 Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Tue, 2 Nov 2021 01:57:38 +0900 Subject: [PATCH 5/7] test: Run test_watch_wildcard_path_without_watch_timer only on Linux Signed-off-by: Takuro Ashie --- test/plugin/test_in_tail.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb index c312be54d1..9f6a50f132 100644 --- a/test/plugin/test_in_tail.rb +++ b/test/plugin/test_in_tail.rb @@ -598,11 +598,14 @@ def test_emit_without_watch_timer(data) # https://github.com/fluent/fluentd/pull/3541#discussion_r740197711 def test_watch_wildcard_path_without_watch_timer + omit "need inotify" unless Fluent.linux? + config = config_element("ROOT", "", { "path" => "#{TMP_DIR}/tail*.txt", "tag" => "t1", }) config = config + CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG + File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test1" f.puts "test2" From 200bd27f7ca2bd9fa2cbc12694e9d98375a6093b Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Tue, 2 Nov 2021 09:18:06 +0900 Subject: [PATCH 6/7] in_tail: Avoid to build debug string when debug mode isn't activated Signed-off-by: Takuro Ashie --- lib/fluent/plugin/in_tail.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb index efeed94af2..413ee4ec39 100644 --- a/lib/fluent/plugin/in_tail.rb +++ b/lib/fluent/plugin/in_tail.rb @@ -370,9 +370,11 @@ def refresh_watchers target_paths_hash = expand_paths existence_paths_hash = existence_path - target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",") - existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",") - log.debug { "tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}" } + log.debug { + target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",") + existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",") + "tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}" + } unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)} added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)} From ebfd278cb5732c9afadeaa18866c323d396a1431 Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Tue, 2 Nov 2021 09:26:20 +0900 Subject: [PATCH 7/7] test_in_tail: Add timeout for test_watch_wildcard_path_without_watch_timer Signed-off-by: Takuro Ashie --- test/plugin/test_in_tail.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb index 9f6a50f132..110421fec3 100644 --- a/test/plugin/test_in_tail.rb +++ b/test/plugin/test_in_tail.rb @@ -613,7 +613,7 @@ def test_watch_wildcard_path_without_watch_timer d = create_driver(config, false) - d.run(expect_emits: 1) do + d.run(expect_emits: 1, timeout: 1) do File.open("#{TMP_DIR}/tail.txt", "ab") {|f| f.puts "test3" f.puts "test4"