From 8f61602369fb19ee8eade9638e97b3d9d23622bc Mon Sep 17 00:00:00 2001
From: Charles-Antoine Mathieu <ca.mathieu@criteo.com>
Date: Mon, 28 Sep 2020 12:03:39 +0200
Subject: [PATCH] S3 backend path prefix #316

---
 server/data/s3/s3.go    | 22 +++++++++++++++-------
 server/plikd.cfg        |  4 +++-
 testing/minio/plikd.cfg |  1 +
 3 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/server/data/s3/s3.go b/server/data/s3/s3.go
index 783cec9b..7b0669bc 100644
--- a/server/data/s3/s3.go
+++ b/server/data/s3/s3.go
@@ -21,6 +21,7 @@ type Config struct {
 	SecretAccessKey string
 	Bucket          string
 	Location        string
+	Prefix          string
 	PartSize        uint64
 	UseSSL          bool
 }
@@ -31,7 +32,7 @@ func NewConfig(params map[string]interface{}) (config *Config) {
 	config = new(Config)
 	config.Bucket = "plik"
 	config.Location = "us-east-1"
-	config.PartSize = 32 * 1024 * 1024 // 32MB
+	config.PartSize = 16 * 1000 * 1000 // 16MB
 	utils.Assign(config, params)
 	return
 }
@@ -53,7 +54,7 @@ func (config *Config) Validate() error {
 	if config.Location == "" {
 		return fmt.Errorf("missing location")
 	}
-	if config.PartSize < 5*1024*1024 {
+	if config.PartSize < 5*1000*1000 {
 		return fmt.Errorf("invalid part size")
 	}
 	return nil
@@ -100,24 +101,31 @@ func NewBackend(config *Config) (b *Backend, err error) {
 
 // GetFile implementation for S3 Data Backend
 func (b *Backend) GetFile(file *common.File) (reader io.ReadCloser, err error) {
-	return b.client.GetObject(b.config.Bucket, file.ID, minio.GetObjectOptions{})
+	return b.client.GetObject(b.config.Bucket, b.getObjectName(file.ID), minio.GetObjectOptions{})
 }
 
 // AddFile implementation for S3 Data Backend
 func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (backendDetails string, err error) {
 	if file.Size > 0 {
-		_, err = b.client.PutObject(b.config.Bucket, file.ID, fileReader, file.Size, minio.PutObjectOptions{ContentType: file.Type})
+		_, err = b.client.PutObject(b.config.Bucket, b.getObjectName(file.ID), fileReader, file.Size, minio.PutObjectOptions{ContentType: file.Type})
 	} else {
 		// https://github.com/minio/minio-go/issues/989
 		// Minio defaults to 128MB chunks and has to actually allocate a buffer of this size before uploading the chunk
 		// This can lead to very high memory usage when uploading a lot of small files in parallel
-		// We default to 32MB which allow to store files up to 320GB ( 10000 chunks of 32MB ), feel free to adjust this parameter to your needs.
-		_, err = b.client.PutObject(b.config.Bucket, file.ID, fileReader, -1, minio.PutObjectOptions{ContentType: file.Type, PartSize: b.config.PartSize})
+		// We default to 16MB which allow to store files up to 160GB ( 10000 chunks of 16MB ), feel free to adjust this parameter to your needs.
+		_, err = b.client.PutObject(b.config.Bucket, b.getObjectName(file.ID), fileReader, -1, minio.PutObjectOptions{ContentType: file.Type, PartSize: b.config.PartSize})
 	}
 	return "", err
 }
 
 // RemoveFile implementation for S3 Data Backend
 func (b *Backend) RemoveFile(file *common.File) (err error) {
-	return b.client.RemoveObject(b.config.Bucket, file.ID)
+	return b.client.RemoveObject(b.config.Bucket, b.getObjectName(file.ID))
 }
+
+func (b *Backend) getObjectName(name string) string {
+	if b.config.Prefix != "" {
+		return fmt.Sprintf("%s/%s", b.config.Prefix, name)
+	}
+	return name
+}
\ No newline at end of file
diff --git a/server/plikd.cfg b/server/plikd.cfg
index 4df043fd..96b76375 100644
--- a/server/plikd.cfg
+++ b/server/plikd.cfg
@@ -68,8 +68,10 @@ OvhApiEndpoint      = ""            # OVH api endpoint to use. Defaults to https
 #       SecretAccessKey = "access_key_secret"
 #       Bucket = "plik"
 #       Location = "us-east-1"
+#       Prefix = ""
 #       UseSSL = true
-#       PartSize = 33554432 // Chunk size when file size is not known ( default to 32MB )
+#       PartSize = 16000000 // Chunk size when file size is not known. (default to 16MB)
+#                           // Multiply by 10000 to get the max upload file size (max upload file size 160GB)
 
 DataBackend = "file"
 [DataBackendConfig]
diff --git a/testing/minio/plikd.cfg b/testing/minio/plikd.cfg
index cb4cfe00..39d6f491 100644
--- a/testing/minio/plikd.cfg
+++ b/testing/minio/plikd.cfg
@@ -12,6 +12,7 @@ DataBackend  = "s3"
     Endpoint = "127.0.0.1:2604"
     AccessKeyID = "access_key"
     SecretAccessKey = "access_key_secret"
+    Prefix = ""
     PartSize = 5242880 # 5MB
 
 [MetadataBackendConfig]