From 3878f7ca60abac78520a79fdd5770e6b27e3e5dd Mon Sep 17 00:00:00 2001
From: William Orr <will@worrbase.com>
Date: Fri, 23 Oct 2020 00:21:58 +0200
Subject: [PATCH] Use a more resilient s3 object name by default

The s3 plugin uses a default object key that is problematic in a few ways.

1. It makes HEAD requests for each chunk it uploads, starting from 1 each time.
If you have uploaded 2000 log files within the same time slice, it will make
2001 HEAD requests to figure out if it exists.

https://github.com/fluent/fluent-plugin-s3/issues/160

2. The above check is not thread-safe, and two threads can race and decide to
use the same `%{index}` value, with the loser of the race overwriting the chunk
from the winner.

https://github.com/fluent/fluent-plugin-s3/issues/326

This is planned to change for v2, but there's no clear path to v2 right now.
The plugin does warn already if you use multiple threads and don't use either
`%{chunk_id}` or `%{uuid_hash}` in the object key.
---
 pkg/sdk/model/output/s3.go          | 4 ++--
 pkg/sdk/model/output/s3_test.go     | 1 +
 pkg/sdk/model/render/fluent_test.go | 3 +++
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/pkg/sdk/model/output/s3.go b/pkg/sdk/model/output/s3.go
index c55db0b4a..3a99a358b 100644
--- a/pkg/sdk/model/output/s3.go
+++ b/pkg/sdk/model/output/s3.go
@@ -127,8 +127,8 @@ type S3OutputConfig struct {
 	GrantFullControl string `json:"grant_full_control,omitempty"`
 	// The length of `%{hex_random}` placeholder(4-16)
 	HexRandomLength string `json:"hex_random_length,omitempty"`
-	// The format of S3 object keys (default: %{path}%{time_slice}_%{index}.%{file_extension})
-	S3ObjectKeyFormat string `json:"s3_object_key_format,omitempty"`
+	// The format of S3 object keys (default: %{path}%{time_slice}_%{uuid_hash}_%{index}.%{file_extension})
+	S3ObjectKeyFormat string `json:"s3_object_key_format,omitempty" plugin:"default:%{path}%{time_slice}_%{uuid_hash}_%{index}.%{file_extension}"`
 	// S3 bucket name
 	S3Bucket string `json:"s3_bucket"`
 	// Archive format on S3
diff --git a/pkg/sdk/model/output/s3_test.go b/pkg/sdk/model/output/s3_test.go
index b8bb50351..5c763d878 100644
--- a/pkg/sdk/model/output/s3_test.go
+++ b/pkg/sdk/model/output/s3_test.go
@@ -39,6 +39,7 @@ buffer:
     @id test
     path logs/${tag}/%Y/%m/%d/
     s3_bucket logging-amazon-s3
+    s3_object_key_format %{path}%{time_slice}_%{uuid_hash}_%{index}.%{file_extension}
     s3_region eu-central-1
     <buffer tag,time>
       @type file
diff --git a/pkg/sdk/model/render/fluent_test.go b/pkg/sdk/model/render/fluent_test.go
index b9c58acf0..abc8d4c8d 100644
--- a/pkg/sdk/model/render/fluent_test.go
+++ b/pkg/sdk/model/render/fluent_test.go
@@ -479,6 +479,7 @@ func TestRenderS3(t *testing.T) {
                         @id test
 						path /var/buffer
 						s3_bucket test_bucket
+                                                s3_object_key_format %{path}%{time_slice}_%{uuid_hash}_%{index}.%{file_extension}
 						<buffer tag,time>
 						@type file
 						path asd
@@ -502,6 +503,7 @@ func TestRenderS3(t *testing.T) {
                         @id test
 						path /var/buffer
 						s3_bucket test_bucket
+                                                s3_object_key_format %{path}%{time_slice}_%{uuid_hash}_%{index}.%{file_extension}
 						<instance_profile_credentials>
 						</instance_profile_credentials>`,
 		},
@@ -519,6 +521,7 @@ func TestRenderS3(t *testing.T) {
                         @id test
 						path /var/buffer
 						s3_bucket test_bucket
+                                                s3_object_key_format %{path}%{time_slice}_%{uuid_hash}_%{index}.%{file_extension}
 						<shared_credentials>
 							path e
 							profile_name f