From 7968c96b89862e831b60fa1d4d1719b31a08d7a0 Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Mon, 27 Dec 2021 23:43:46 +0900 Subject: [PATCH 1/5] GitHub Actions: Add Ruby 3.1 for Windows Signed-off-by: Takuro Ashie --- .github/workflows/windows-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows-test.yaml b/.github/workflows/windows-test.yaml index 1f43f18e57..9de1405f80 100644 --- a/.github/workflows/windows-test.yaml +++ b/.github/workflows/windows-test.yaml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ['2.7', '2.6'] + ruby-version: [3.1, '2.7', '2.6'] os: - windows-latest experimental: [false] From 4f668f935193afb52a975208ac129f15b1cdcad0 Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Fri, 18 Feb 2022 22:16:39 +0900 Subject: [PATCH 2/5] CI: Run only Ruby 3.1 on Windows to investigate issues on it Signed-off-by: Takuro Ashie --- .drone.yml | 35 ---------- .github/workflows/linux-test.yaml | 36 ---------- .github/workflows/macos-test.yaml | 34 --------- .github/workflows/windows-test.yaml | 19 +---- .gitlab-ci.yml | 103 ---------------------------- 5 files changed, 2 insertions(+), 225 deletions(-) delete mode 100644 .drone.yml delete mode 100644 .github/workflows/linux-test.yaml delete mode 100644 .github/workflows/macos-test.yaml delete mode 100644 .gitlab-ci.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 4a2a84c06d..0000000000 --- a/.drone.yml +++ /dev/null @@ -1,35 +0,0 @@ -kind: pipeline -name: fluentd-test-arm64-3-0 - -platform: - os: linux - arch: arm64 - -steps: -- name: fluentd-test-arm64-3-0 - image: arm64v8/ruby:3.0 - commands: - - apt update - - apt -y install libgmp3-dev - - export BUNDLE_GEMFILE=$PWD/Gemfile - - gem update --system - - bundle install --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle} - - bundle exec rake test ---- -kind: pipeline -name: fluentd-test-arm64-2-7 - -platform: - os: linux - arch: arm64 - -steps: -- name: fluentd-test-arm64-2-7 - image: arm64v8/ruby:2.7 - commands: - - apt update - - apt -y install libgmp3-dev - - export BUNDLE_GEMFILE=$PWD/Gemfile - - gem update --system - - bundle install --jobs=3 --retry=3 --path=${BUNDLE_PATH:-vendor/bundle} - - bundle exec rake test diff --git a/.github/workflows/linux-test.yaml b/.github/workflows/linux-test.yaml deleted file mode 100644 index f814623805..0000000000 --- a/.github/workflows/linux-test.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: Testing on Ubuntu - -on: - push: - branches: [master] - pull_request: - branches: [master] - -jobs: - test: - runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.experimental }} - strategy: - fail-fast: false - matrix: - ruby-version: ['3.1', '3.0', '2.7', '2.6'] - os: [ubuntu-latest] - experimental: [false] - include: - - ruby-version: head - os: ubuntu-latest - experimental: true - - name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby-version }} - - name: Install addons - run: sudo apt-get install libgmp3-dev libcap-ng-dev - - name: Install dependencies - run: bundle install - - name: Run tests - run: bundle exec rake test diff --git a/.github/workflows/macos-test.yaml b/.github/workflows/macos-test.yaml deleted file mode 100644 index 630b03de7a..0000000000 --- a/.github/workflows/macos-test.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Testing on macOS - -on: - push: - branches: [master] - pull_request: - branches: [master] - -jobs: - test: - runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.experimental }} - strategy: - fail-fast: false - matrix: - ruby-version: ['3.1', '3.0', '2.7'] - os: [macos-latest] - experimental: [true] - include: - - ruby-version: head - os: macos-latest - experimental: true - - name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - - name: Set up Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby-version }} - - name: Install dependencies - run: bundle install - - name: Run tests - run: bundle exec rake test diff --git a/.github/workflows/windows-test.yaml b/.github/workflows/windows-test.yaml index 9de1405f80..d20893ba30 100644 --- a/.github/workflows/windows-test.yaml +++ b/.github/workflows/windows-test.yaml @@ -13,22 +13,10 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: [3.1, '2.7', '2.6'] + ruby-version: ['3.1'] os: - windows-latest experimental: [false] - include: - - ruby-version: '3.0.3' - os: windows-latest - experimental: false - # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct - # error code. In addition, we have to specify the path of fiddle by RUBYLIB - # because RubyInstaller loads Ruby's bundled fiddle before initializing gem. - # See also: - # * https://github.com/ruby/fiddle/issues/72 - # * https://bugs.ruby-lang.org/issues/17813 - # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb - ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.3/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.1.0/lib name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }} steps: @@ -37,10 +25,7 @@ jobs: uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby-version }} - - name: Add Fiddle 1.1.0 - if: ${{ matrix.ruby-version == '3.0.3' }} - run: gem install fiddle --version 1.1.0 - name: Install dependencies run: ridk exec bundle install - name: Run tests - run: bundle exec rake test TESTOPTS=-v ${{ matrix.ruby-lib-opt }} + run: bundle exec rake test TESTOPTS=-v diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index d1e0de3095..0000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,103 +0,0 @@ -.install-template: - before_script: - - gem update --system - - ruby -v - -.test-template: - before_script: - - gem install rake - - bundle install - - chmod 777 -R ./test - -stages: - - build - - tests - -2-6-6: - image: "ruby:2.6.6" - stage: build - extends: .install-template - script: - - bundle install --jobs=3 --retry=3 - cache: - key: "$CI_COMMIT_REF_SLUG 2-6-6" - paths: - -2-7-2: - image: "ruby:2.7.2" - stage: build - extends: .install-template - script: - - bundle install --jobs=3 --retry=3 - cache: - key: "$CI_COMMIT_REF_SLUG 2-7-2" - paths: - -3-0-0: - image: "ruby:3.0.0" - stage: build - extends: .install-template - script: - - bundle install --jobs=3 --retry=3 - cache: - key: "$CI_COMMIT_REF_SLUG 3-0-0" - paths: - -rubyhead: - image: "ruby:latest" - stage: build - extends: .install-template - script: - - bundle install --jobs=3 --retry=3 - cache: - key: "$CI_COMMIT_REF_SLUG latest" - paths: - - ./* - -2-6-6-test: - image: "ruby:2.6.6" - stage: tests - allow_failure: true - extends: .test-template - script: - - bundle exec rake test - cache: - key: "$CI_COMMIT_REF_SLUG 2-6-6" - paths: - - ./* - -2-7-2-test: - image: "ruby:2.7.2" - stage: tests - allow_failure: true - extends: .test-template - script: - - bundle exec rake test - cache: - key: "$CI_COMMIT_REF_SLUG 2-7-2" - paths: - - ./* - -3-0-0-test: - image: "ruby:3.0.0" - stage: tests - allow_failure: true - extends: .test-template - script: - - bundle exec rake test - cache: - key: "$CI_COMMIT_REF_SLUG 3-0-0" - paths: - - ./* - -rubyhead-test: - image: "ruby:latest" - stage: tests - allow_failure: true - extends: .test-template - script: - - bundle exec rake test - cache: - key: "$CI_COMMIT_REF_SLUG latest" - paths: - - ./* From 1117d8c4508c44d42e1f765bc36d74f33988a079 Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Fri, 18 Feb 2022 22:38:38 +0900 Subject: [PATCH 3/5] CI: Separate in_tail test Because in_tail test seems completely broken and it breaks also other test. It's too noisy to investivate other failed tests. Signed-off-by: Takuro Ashie --- .github/workflows/windows-test.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/windows-test.yaml b/.github/workflows/windows-test.yaml index d20893ba30..937d69d481 100644 --- a/.github/workflows/windows-test.yaml +++ b/.github/workflows/windows-test.yaml @@ -1,4 +1,4 @@ -name: Testing on Windows +name: Ruby 3.1 on Windows on: push: @@ -14,11 +14,12 @@ jobs: fail-fast: false matrix: ruby-version: ['3.1'] + in_tail: [true, false] os: - windows-latest experimental: [false] - name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }} + name: in_tail ${{ matrix.in_tail }} steps: - uses: actions/checkout@v2 - name: Set up Ruby @@ -27,5 +28,9 @@ jobs: ruby-version: ${{ matrix.ruby-version }} - name: Install dependencies run: ridk exec bundle install - - name: Run tests - run: bundle exec rake test TESTOPTS=-v + - name: test in_tail + if: ${{ matrix.in_tail }} + run: bundle exec rake test TESTOPTS="-v --testcase='/(TailInputTest|IntailIOHandlerTest)/'" + - name: test without in_tail + if: ${{ !matrix.in_tail }} + run: bundle exec rake test TESTOPTS="-v --ignore-testcase='/(TailInputTest|IntailIOHandlerTest)/'" From ddf0c3d0e59b786670b82a913c1fd521dfefafbc Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Sat, 19 Feb 2022 01:23:00 +0900 Subject: [PATCH 4/5] test_out_forward: Use assert_rr For avoiding the following error: ``` 2022-02-18T01:36:00.3691879Z =============================================================================== 2022-02-18T01:36:00.3692453Z Failure: test: send tags in str (utf-8 strings)(ForwardOutputTest): 2022-02-18T01:36:00.3692799Z On subject , 2022-02-18T01:36:00.3693302Z unexpected method invocation: 2022-02-18T01:36:00.3693595Z process_wait(2088, 0) 2022-02-18T01:36:00.3702794Z expected invocations: 2022-02-18T01:36:00.3703495Z C:/hostedtoolcache/windows/Ruby/3.1.0/x64/lib/ruby/gems/3.1.0/gems/serverengine-2.2.5/lib/serverengine/socket_manager.rb:79:in ``' 2022-02-18T01:36:00.3704338Z C:/hostedtoolcache/windows/Ruby/3.1.0/x64/lib/ruby/gems/3.1.0/gems/serverengine-2.2.5/lib/serverengine/socket_manager.rb:79:in `block in generate_path' 2022-02-18T01:36:00.3705025Z C:/hostedtoolcache/windows/Ruby/3.1.0/x64/lib/ruby/gems/3.1.0/gems/serverengine-2.2.5/lib/serverengine/socket_manager.rb:78:in `each' 2022-02-18T01:36:00.3705695Z C:/hostedtoolcache/windows/Ruby/3.1.0/x64/lib/ruby/gems/3.1.0/gems/serverengine-2.2.5/lib/serverengine/socket_manager.rb:78:in `generate_path' 2022-02-18T01:36:00.3706156Z D:/a/fluentd/fluentd/lib/fluent/test/driver/base.rb:105:in `instance_start' 2022-02-18T01:36:00.3706550Z D:/a/fluentd/fluentd/lib/fluent/test/driver/base.rb:77:in `run' 2022-02-18T01:36:00.3706945Z D:/a/fluentd/fluentd/lib/fluent/test/driver/base_owner.rb:130:in `run' 2022-02-18T01:36:00.3707422Z D:/a/fluentd/fluentd/test/plugin/test_out_forward.rb:434:in `block in ' 2022-02-18T01:36:00.3707780Z 431: ] 2022-02-18T01:36:00.3708021Z 432: 2022-02-18T01:36:00.3708366Z 433: stub(d.instance.ack_handler).read_ack_from_sock(anything).never 2022-02-18T01:36:00.3708758Z => 434: target_input_driver.run(expect_records: 2) do 2022-02-18T01:36:00.3709053Z 435: d.run do 2022-02-18T01:36:00.3709364Z 436: emit_events.each do |tag, t, record| 2022-02-18T01:36:00.3709681Z 437: d.feed(tag, t, record) 2022-02-18T01:36:00.3709997Z =============================================================================== ``` Signed-off-by: Takuro Ashie --- test/plugin/test_out_forward.rb | 50 ++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/test/plugin/test_out_forward.rb b/test/plugin/test_out_forward.rb index 53aca518c2..66a2776cc5 100644 --- a/test/plugin/test_out_forward.rb +++ b/test/plugin/test_out_forward.rb @@ -431,10 +431,12 @@ def try_write(chunk) ] stub(d.instance.ack_handler).read_ack_from_sock(anything).never - target_input_driver.run(expect_records: 2) do - d.run do - emit_events.each do |tag, t, record| - d.feed(tag, t, record) + assert_rr do + target_input_driver.run(expect_records: 2) do + d.run do + emit_events.each do |tag, t, record| + d.feed(tag, t, record) + end end end end @@ -461,10 +463,12 @@ def try_write(chunk) ] stub(d.instance.ack_handler).read_ack_from_sock(anything).never - target_input_driver.run(expect_records: 2) do - d.run(default_tag: 'test') do - records.each do |record| - d.feed(time, record) + assert_rr do + target_input_driver.run(expect_records: 2) do + d.run(default_tag: 'test') do + records.each do |record| + d.feed(time, record) + end end end end @@ -491,10 +495,12 @@ def try_write(chunk) {"a" => 2} ] stub(d.instance.ack_handler).read_ack_from_sock(anything).never - target_input_driver.run(expect_records: 2) do - d.run(default_tag: 'test') do - records.each do |record| - d.feed(time, record) + assert_rr do + target_input_driver.run(expect_records: 2) do + d.run(default_tag: 'test') do + records.each do |record| + d.feed(time, record) + end end end end @@ -549,10 +555,12 @@ def try_write(chunk) ] # not attempt to receive responses stub(d.instance.ack_handler).read_ack_from_sock(anything).never - target_input_driver.run(expect_records: 2) do - d.run(default_tag: 'test') do - records.each do |record| - d.feed(time, record) + assert_rr do + target_input_driver.run(expect_records: 2) do + d.run(default_tag: 'test') do + records.each do |record| + d.feed(time, record) + end end end end @@ -575,10 +583,12 @@ def try_write(chunk) ] # not attempt to receive responses stub(d.instance.ack_handler).read_ack_from_sock(anything).never - target_input_driver.run(expect_records: 2) do - d.run(default_tag: 'test') do - records.each do |record| - d.feed(time, record) + assert_rr do + target_input_driver.run(expect_records: 2) do + d.run(default_tag: 'test') do + records.each do |record| + d.feed(time, record) + end end end end From d565510a8a0be567e84463dc2bf12b622151f3bf Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Mon, 30 May 2022 18:12:13 +0900 Subject: [PATCH 5/5] WIP: Use File::Constants::SHARE_DELETE instead of FileWrapper Signed-off-by: Takuro Ashie --- lib/fluent/plugin/in_tail.rb | 2 +- test/plugin/test_in_tail.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fluent/plugin/in_tail.rb b/lib/fluent/plugin/in_tail.rb index deea7c9498..538f325cb5 100644 --- a/lib/fluent/plugin/in_tail.rb +++ b/lib/fluent/plugin/in_tail.rb @@ -1117,7 +1117,7 @@ def handle_notify end def open - io = Fluent::FileWrapper.open(@path) + io = File.open(@path, File::Constants::RDWR | File::Constants::BINARY | File::Constants::SHARE_DELETE) io.seek(@watcher.pe.read_pos + @fifo.bytesize) @metrics.opened.inc io diff --git a/test/plugin/test_in_tail.rb b/test/plugin/test_in_tail.rb index 36abb25a6c..0335384759 100644 --- a/test/plugin/test_in_tail.rb +++ b/test/plugin/test_in_tail.rb @@ -999,7 +999,7 @@ def test_rotate_file_with_write_old_and_no_new_file(data) end def sub_test_rotate_file(config = nil, expect_emits: nil, expect_records: nil, timeout: nil) - file = Fluent::FileWrapper.open("#{TMP_DIR}/tail.txt", "wb") + file = File.open("#{TMP_DIR}/tail.txt", File::Constants::WRONLY | File::Constants::CREAT | File::Constants::TRUNC | File::Constants::BINARY | File::Constants::SHARE_DELETE) file.puts "test1" file.puts "test2" file.flush