Merge pull request #3579 from yaojingguo/workaround-for-out-file-appe…

…nd-option-without-compression Make out_file append option work without compression
fluent · Dec 28, 2021 · 362ff08 · 362ff08
2 parents 8229ed9 + f17cb9e
commit 362ff08
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 14 deletions.
diff --git a/lib/fluent/env.rb b/lib/fluent/env.rb
@@ -33,4 +33,8 @@ def self.windows?
   def self.linux?
     /linux/ === RUBY_PLATFORM
   end
+
+  def self.macos?
+    /darwin/ =~ RUBY_PLATFORM
+  end
 end
diff --git a/lib/fluent/plugin/out_file.rb b/lib/fluent/plugin/out_file.rb
@@ -181,6 +181,13 @@ def configure(conf)
       @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
       @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
       @need_lock = system_config.workers > 1
+
+      # https://github.com/fluent/fluentd/issues/3569
+      @need_ruby_on_macos_workaround = false
+      if @append && Fluent.macos?
+        condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
+        @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
+      end
     end
 
     def multi_workers_ready?
@@ -223,7 +230,12 @@ def write(chunk)
 
     def write_without_compression(path, chunk)
       File.open(path, "ab", @file_perm) do |f|
-        chunk.write_to(f)
+        if @need_ruby_on_macos_workaround
+          content = chunk.read()
+          f.puts content
+        else
+          chunk.write_to(f)
+        end
       end
     end
 

diff --git a/test/plugin/test_out_file.rb b/test/plugin/test_out_file.rb
@@ -394,6 +394,11 @@ def check_gzipped_result(path, expect)
     assert_equal expect, result
   end
 
+  def check_result(path, expect)
+    result = File.read(path, mode: "rb")
+    assert_equal expect, result
+  end
+
   sub_test_case 'write' do
     test 'basic case' do
       d = create_driver
@@ -535,38 +540,49 @@ def parse_system(text)
     assert_equal 3, Dir.glob("#{TMP_DIR}/out_file_test.*").size
   end
 
-  test 'append' do
+  data(
+    "with compression" => true,
+    "without compression" => false,
+  )
+  test 'append' do |compression|
     time = event_time("2011-01-02 13:14:15 UTC")
     formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]
 
     write_once = ->(){
-      d = create_driver %[
+      config = %[
         path #{TMP_DIR}/out_file_test
-        compress gz
         utc
         append true
         <buffer>
           timekey_use_utc true
         </buffer>
       ]
+      if compression
+        config << "        compress gz"
+      end
+      d = create_driver(config)
       d.run(default_tag: 'test'){
         d.feed(time, {"a"=>1})
         d.feed(time, {"a"=>2})
       }
       d.instance.last_written_path
     }
 
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines)
-
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 2)
+    log_file_name = "out_file_test.20110102.log"
+    if compression
+      log_file_name << ".gz"
+    end
 
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 3)
+    1.upto(3) do |i|
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/#{log_file_name}", path
+      expect = formatted_lines * i
+      if compression
+        check_gzipped_result(path, expect)
+      else
+        check_result(path, expect)
+      end
+    end
   end
 
   test 'append when JST' do