From bf8bb2c9b1bf49ddb34e77e76707e3c180bf86a5 Mon Sep 17 00:00:00 2001 From: daipom Date: Wed, 9 Mar 2022 16:37:42 +0900 Subject: [PATCH] Add testcode for multiple workers f55d70f51 added a dataloss warning for multiple workers. This patch adds tests for the case. Note that you can execute the exact test case by running: $ bundle exec rake test TESTOPTS="-t'S3OutputTest::configure'" Signed-off-by: daipom Reviewed-by: Fujimoto Seiji --- test/test_out_s3.rb | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/test_out_s3.rb b/test/test_out_s3.rb index 54af2c7..b00bd3c 100644 --- a/test/test_out_s3.rb +++ b/test/test_out_s3.rb @@ -150,6 +150,49 @@ def test_configure_with_grant assert_equal "id='2345678901'", d.instance.grant_read_acp assert_equal "id='3456789012'", d.instance.grant_write_acp end + + CONFIG_WITH_OBJECTKEY_DEFAULT = %[ + s3_object_key_format "%{path}%{time_slice}_%{index}.%{file_extension}" + aws_key_id test_key_id + aws_sec_key test_sec_key + s3_bucket test_bucket + path log + utc + buffer_type memory + time_slice_format %Y%m%d-%H + ] + + CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD = %[ + s3_object_key_format "%{path}%{time_slice}_${chunk_id}.%{file_extension}" + aws_key_id test_key_id + aws_sec_key test_sec_key + s3_bucket test_bucket + path log + utc + buffer_type memory + time_slice_format %Y%m%d-%H + ] + + data("non_objectkey", {"expected_warning_num" => 1, "conf" => CONFIG, "workers" => 1, "with_multi_buffers" => false}) + data("non_objectkey-multi_buffer", {"expected_warning_num" => 2, "conf" => CONFIG, "workers" => 1, "with_multi_buffers" => true}) + data("non_objectkey-multi_worker", {"expected_warning_num" => 2, "conf" => CONFIG, "workers" => 2, "with_multi_buffers" => false}) + data("default_objectkey", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 1, "with_multi_buffers" => false}) + data("default_objectkey-multi_buffer", {"expected_warning_num" => 1, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 1, "with_multi_buffers" => true}) + data("default_objectkey-multi_worker", {"expected_warning_num" => 1, "conf" => CONFIG_WITH_OBJECTKEY_DEFAULT, "workers" => 2, "with_multi_buffers" => false}) + data("fixed_objectkey", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 1, "with_multi_buffers" => false}) + data("fixed_objectkey-multi_buffer", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 1, "with_multi_buffers" => true}) + data("fixed_objectkey-multi_worker", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 2, "with_multi_buffers" => false}) + def test_configure_warning_on_parallel(data) + conf = data["conf"].clone + if data["with_multi_buffers"] + conf << "\n\n@type memory\nflush_thread_count 2\n\n" + end + assert_rr do + d = Fluent::Test::Driver::Output.new(Fluent::Plugin::S3Output, opts: {"workers": data["workers"]}) + mock(d.instance.log).warn(anything).times(data["expected_warning_num"]) + d.configure(conf) + end + end end def test_format