S3 backend path prefix root-gg#316

Author: Charles-Antoine Mathieu

server/data/s3/s3.go

@@ -21,6 +21,7 @@ type Config struct {
 	SecretAccessKey string
 	Bucket          string
 	Location        string
+	Prefix          string
 	PartSize        uint64
 	UseSSL          bool
 }
@@ -31,7 +32,7 @@ func NewConfig(params map[string]interface{}) (config *Config) {
 	config = new(Config)
 	config.Bucket = "plik"
 	config.Location = "us-east-1"
-	config.PartSize = 32 * 1024 * 1024 // 32MB
+	config.PartSize = 16 * 1000 * 1000 // 16MB
 	utils.Assign(config, params)
 	return
 }
@@ -53,7 +54,7 @@ func (config *Config) Validate() error {
 	if config.Location == "" {
 		return fmt.Errorf("missing location")
 	}
-	if config.PartSize < 5*1024*1024 {
+	if config.PartSize < 5*1000*1000 {
 		return fmt.Errorf("invalid part size")
 	}
 	return nil
@@ -100,24 +101,31 @@ func NewBackend(config *Config) (b *Backend, err error) {
 
 // GetFile implementation for S3 Data Backend
 func (b *Backend) GetFile(file *common.File) (reader io.ReadCloser, err error) {
-	return b.client.GetObject(b.config.Bucket, file.ID, minio.GetObjectOptions{})
+	return b.client.GetObject(b.config.Bucket, b.getObjectName(file.ID), minio.GetObjectOptions{})
 }
 
 // AddFile implementation for S3 Data Backend
 func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (backendDetails string, err error) {
 	if file.Size > 0 {
-		_, err = b.client.PutObject(b.config.Bucket, file.ID, fileReader, file.Size, minio.PutObjectOptions{ContentType: file.Type})
+		_, err = b.client.PutObject(b.config.Bucket, b.getObjectName(file.ID), fileReader, file.Size, minio.PutObjectOptions{ContentType: file.Type})
 	} else {
 		// https://github.com/minio/minio-go/issues/989
 		// Minio defaults to 128MB chunks and has to actually allocate a buffer of this size before uploading the chunk
 		// This can lead to very high memory usage when uploading a lot of small files in parallel
-		// We default to 32MB which allow to store files up to 320GB ( 10000 chunks of 32MB ), feel free to adjust this parameter to your needs.
-		_, err = b.client.PutObject(b.config.Bucket, file.ID, fileReader, -1, minio.PutObjectOptions{ContentType: file.Type, PartSize: b.config.PartSize})
+		// We default to 16MB which allow to store files up to 160GB ( 10000 chunks of 16MB ), feel free to adjust this parameter to your needs.
+		_, err = b.client.PutObject(b.config.Bucket, b.getObjectName(file.ID), fileReader, -1, minio.PutObjectOptions{ContentType: file.Type, PartSize: b.config.PartSize})
 	}
 	return "", err
 }
 
 // RemoveFile implementation for S3 Data Backend
 func (b *Backend) RemoveFile(file *common.File) (err error) {
-	return b.client.RemoveObject(b.config.Bucket, file.ID)
+	return b.client.RemoveObject(b.config.Bucket, b.getObjectName(file.ID))
+}
+
+func (b *Backend) getObjectName(name string) string {
+	if b.config.Prefix != "" {
+		return fmt.Sprintf("%s/%s", b.config.Prefix, name)
+	}
+	return name
 }

server/plikd.cfg

@@ -68,8 +68,10 @@ OvhApiEndpoint      = ""            # OVH api endpoint to use. Defaults to https
 #       SecretAccessKey = "access_key_secret"
 #       Bucket = "plik"
 #       Location = "us-east-1"
+#       Prefix = ""
 #       UseSSL = true
-#       PartSize = 33554432 // Chunk size when file size is not known ( default to 32MB )
+#       PartSize = 16000000 // Chunk size when file size is not known. (default to 16MB)
+#                           // Multiply by 10000 to get the max upload file size (max upload file size 160GB)
 
 DataBackend = "file"
 [DataBackendConfig]

testing/minio/plikd.cfg

@@ -12,6 +12,7 @@ DataBackend  = "s3"
     Endpoint = "127.0.0.1:2604"
     AccessKeyID = "access_key"
     SecretAccessKey = "access_key_secret"
+    Prefix = ""
     PartSize = 5242880 # 5MB
 
 [MetadataBackendConfig]