Skip to content

Commit

Permalink
Fix warning message to cover multiple workers
Browse files Browse the repository at this point in the history
Workers consist of processes, so fix some `multi thread` words too.

Signed-off-by: daipom <[email protected]>
  • Loading branch information
daipom committed Mar 10, 2022
1 parent 9570e78 commit 0cdb512
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 4 deletions.
6 changes: 3 additions & 3 deletions lib/fluent/plugin/out_s3.rb
Original file line number Diff line number Diff line change
Expand Up @@ -461,9 +461,9 @@ def check_s3_path_safety(conf)
log.warn "The default value of s3_object_key_format will use ${chunk_id} instead of %{index} to avoid object conflict in v2"
end

is_working_on_multi_thread = @buffer_config.flush_thread_count > 1 || system_config.workers > 1
if is_working_on_multi_thread && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
is_working_on_parallel = @buffer_config.flush_thread_count > 1 || system_config.workers > 1
if is_working_on_parallel && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads or multiple workers. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
end
end

Expand Down
2 changes: 1 addition & 1 deletion test/test_out_s3.rb
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ def test_configure_with_grant
data("fixed_objectkey", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 1, "with_multi_buffers" => false})
data("fixed_objectkey-multi_buffer", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 1, "with_multi_buffers" => true})
data("fixed_objectkey-multi_worker", {"expected_warning_num" => 0, "conf" => CONFIG_WITH_OBJECTKEY_FIXED_FOR_MULTI_THEAD, "workers" => 2, "with_multi_buffers" => false})
def test_configure_warning_on_multi_threads(data)
def test_configure_warning_on_parallel(data)
conf = data["conf"].clone
if data["with_multi_buffers"]
conf << "\n<buffer>\n@type memory\nflush_thread_count 2\n</buffer>\n"
Expand Down

0 comments on commit 0cdb512

Please sign in to comment.