Skip to content

Commit

Permalink
Add warning for object conflict case. ref #326
Browse files Browse the repository at this point in the history
Signed-off-by: Masahiro Nakagawa <[email protected]>
  • Loading branch information
repeatedly committed May 15, 2020
1 parent 7469d77 commit e3d87f1
Showing 1 changed file with 12 additions and 0 deletions.
12 changes: 12 additions & 0 deletions lib/fluent/plugin/out_s3.rb
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,8 @@ def configure(conf)
end
end

check_s3_path_safety(conf)

# For backward compatibility
# TODO: Remove time_slice_format when end of support compat_parameters
@configured_time_slice_format = conf['time_slice_format']
Expand Down Expand Up @@ -451,6 +453,16 @@ def process_s3_object_key_format
}
end

def check_s3_path_safety(conf)
unless conf.has_key?('s3_object_key_format')
log.warn "The default value of s3_object_key_format will use ${chunk_id} instead of %{index} to avoid object conflict in v2"
end

if (@buffer_config.flush_thread_count > 1) && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
end
end

def check_apikeys
@bucket.objects(prefix: @path, :max_keys => 1).first
rescue Aws::S3::Errors::NoSuchBucket
Expand Down

0 comments on commit e3d87f1

Please sign in to comment.