Skip to content

Commit

Permalink
S3 Server Side Encryption
Browse files Browse the repository at this point in the history
  • Loading branch information
Charles-Antoine Mathieu committed Oct 23, 2020
1 parent 9546a83 commit 5e192e0
Show file tree
Hide file tree
Showing 19 changed files with 238 additions and 64 deletions.
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ Plik is a scalable & friendly temporary file upload system ( wetransfer like ) i
- User authentication : Local / Google / OVH
- Upload restriction : Source IP / Token
- Administrator dashboard
- Server side encryption ( with S3 data backend )
- [ShareX](https://getsharex.com/) Uploader : Directly integrated into ShareX
- [plikSharp](https://github.com/iss0/plikSharp) : A .NET API client for Plik
- [Filelink for Plik](https://gitlab.com/joendres/filelink-plik) : Thunderbird Addon to upload attachments to Plik
Expand Down Expand Up @@ -143,7 +144,7 @@ Store uploaded files in a local or mounted file system directory.

- Openstack Swift databackend : http://docs.openstack.org/developer/swift/

Openstack Swift is a highly available, distributed, eventually consistent object/blob store.
Openstack Swift is a highly available, distributed, eventually consistent object/blob store which supports Server Side Encryption

### Available metadata backends

Expand Down
11 changes: 7 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ require (
github.com/fatih/color v1.9.0 // indirect
github.com/golang/protobuf v1.3.2-0.20190409050943-e91709a02e0e // indirect
github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf
github.com/google/uuid v1.1.2 // indirect
github.com/gorilla/mux v1.7.1
github.com/hashicorp/golang-lru v0.5.1 // indirect
github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 // indirect
Expand All @@ -23,7 +24,7 @@ require (
github.com/mattn/go-colorable v0.1.6 // indirect
github.com/mattn/go-runewidth v0.0.5-0.20181218000649-703b5e6b11ae // indirect
github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
github.com/minio/minio-go/v6 v6.0.49
github.com/minio/minio-go/v7 v7.0.5
github.com/mitchellh/go-homedir v1.1.0
github.com/ncw/swift v1.0.48-0.20190410202254-753d2090bb62
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d
Expand All @@ -33,13 +34,15 @@ require (
github.com/root-gg/utils v0.0.0-20151025161626-38f45ede2ce2
github.com/spf13/cobra v0.0.6-0.20200106181057-89c7ffb5129b
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709
github.com/stretchr/testify v1.4.0
go.opencensus.io v0.20.2 // indirect
golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897
golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a
golang.org/x/text v0.3.1-0.20190410012825-f4905fbd45b6 // indirect
golang.org/x/sys v0.0.0-20201020230747-6e5568b54d1a // indirect
google.golang.org/api v0.3.3-0.20190418015003-33b7e862cd15
google.golang.org/appengine v1.5.0 // indirect
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/gormigrate.v1 v1.6.0
gopkg.in/ini.v1 v1.62.0 // indirect
)
62 changes: 50 additions & 12 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion server/data/data.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
// Backend interface describes methods that data backend
// must implements to be compatible with plik.
type Backend interface {
AddFile(file *common.File, reader io.Reader) (backendDetails string, err error)
AddFile(file *common.File, reader io.Reader) (err error)
GetFile(file *common.File) (reader io.ReadCloser, err error)
RemoveFile(file *common.File) (err error)
}
12 changes: 6 additions & 6 deletions server/data/file/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,32 +61,32 @@ func (b *Backend) GetFile(file *common.File) (reader io.ReadCloser, err error) {

// AddFile implementation for file data backend will creates a new file for the given upload
// and save it on filesystem with the given file reader
func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (backendDetails string, err error) {
func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (err error) {
dir, path, err := b.getPath(file)
if err != nil {
return "", err
return err
}

// Create directory
err = os.MkdirAll(dir, 0777)
if err != nil {
return "", fmt.Errorf("unable to create upload directory")
return fmt.Errorf("unable to create upload directory")
}

// Create file
out, err := os.Create(path)
if err != nil {
return "", fmt.Errorf("unable to create file %s : %s", path, err)
return fmt.Errorf("unable to create file %s : %s", path, err)
}

// Copy file data from the client request body
// to the file system
_, err = io.Copy(out, fileReader)
if err != nil {
return "", fmt.Errorf("unable to save file %s : %s", path, err)
return fmt.Errorf("unable to save file %s : %s", path, err)
}

return "", nil
return nil
}

// RemoveFile implementation for file data backend will delete the given
Expand Down
12 changes: 6 additions & 6 deletions server/data/file/file_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func TestAddFileInvalidUploadId(t *testing.T) {
upload := &common.Upload{}
file := upload.NewFile()

_, err := backend.AddFile(file, &bytes.Buffer{})
err := backend.AddFile(file, &bytes.Buffer{})
require.Error(t, err, "no error with invalid upload id")
}

Expand All @@ -56,7 +56,7 @@ func TestAddFileImpossibleToCreateDirectory(t *testing.T) {
file := upload.NewFile()
upload.PrepareInsertForTests()

_, err := backend.AddFile(file, &bytes.Buffer{})
err := backend.AddFile(file, &bytes.Buffer{})
require.Error(t, err, "unable to create directory")
}

Expand All @@ -69,7 +69,7 @@ func TestAddFileInvalidReader(t *testing.T) {
upload.PrepareInsertForTests()

reader := common.NewErrorReader(errors.New("io error"))
_, err := backend.AddFile(file, reader)
err := backend.AddFile(file, reader)
require.Error(t, err, "unable to create directory")
require.Contains(t, err.Error(), "io error", "invalid error")
}
Expand All @@ -83,7 +83,7 @@ func TestAddFile(t *testing.T) {
upload.PrepareInsertForTests()

reader := bytes.NewBufferString("data")
_, err := backend.AddFile(file, reader)
err := backend.AddFile(file, reader)
require.NoError(t, err, "unable to add file")

_, path, err := backend.getPathCompat(file)
Expand Down Expand Up @@ -135,7 +135,7 @@ func TestGetFile(t *testing.T) {
upload.PrepareInsertForTests()

reader := bytes.NewBufferString("data")
_, err := backend.AddFile(file, reader)
err := backend.AddFile(file, reader)
require.NoError(t, err, "unable to add file")

fileReader, err := backend.GetFile(file)
Expand Down Expand Up @@ -215,7 +215,7 @@ func TestRemoveFile(t *testing.T) {
upload.PrepareInsertForTests()

reader := bytes.NewBufferString("data")
_, err := backend.AddFile(file, reader)
err := backend.AddFile(file, reader)
require.NoError(t, err, "unable to add file")

_, path, err := backend.getPathCompat(file)
Expand Down
50 changes: 40 additions & 10 deletions server/data/s3/s3.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package s3

import (
"context"
"fmt"
"io"

"github.com/minio/minio-go/v6"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/root-gg/utils"

"github.com/root-gg/plik/server/common"
Expand All @@ -24,6 +26,7 @@ type Config struct {
Prefix string
PartSize uint64
UseSSL bool
SSE string
}

// NewConfig instantiate a new default configuration
Expand Down Expand Up @@ -60,6 +63,11 @@ func (config *Config) Validate() error {
return nil
}

// BackendDetails additional backend metadata
type BackendDetails struct {
SSEKey string
}

// Backend object
type Backend struct {
config *Config
Expand All @@ -77,20 +85,24 @@ func NewBackend(config *Config) (b *Backend, err error) {
return nil, fmt.Errorf("invalid s3 data backend config : %s", err)
}

b.client, err = minio.New(config.Endpoint, config.AccessKeyID, config.SecretAccessKey, config.UseSSL)
b.client, err = minio.New(config.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
//Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
Secure: config.UseSSL,
})
if err != nil {
return nil, err
}

// Check if bucket exists
exists, err := b.client.BucketExists(config.Bucket)
exists, err := b.client.BucketExists(context.TODO(), config.Bucket)
if err != nil {
return nil, fmt.Errorf("unable to check if bucket %s exists : %s", config.Bucket, err)
}

if !exists {
// Create bucket
err = b.client.MakeBucket(config.Bucket, config.Location)
err = b.client.MakeBucket(context.TODO(), config.Bucket, minio.MakeBucketOptions{Region: config.Location})
if err != nil {
return nil, fmt.Errorf("unable to create bucket %s : %s", config.Bucket, err)
}
Expand All @@ -101,26 +113,44 @@ func NewBackend(config *Config) (b *Backend, err error) {

// GetFile implementation for S3 Data Backend
func (b *Backend) GetFile(file *common.File) (reader io.ReadCloser, err error) {
return b.client.GetObject(b.config.Bucket, b.getObjectName(file.ID), minio.GetObjectOptions{})
getOpts := minio.GetObjectOptions{}

// Configure server side encryption
getOpts.ServerSideEncryption, err = b.getServerSideEncryption(file)
if err != nil {
return nil, err
}

return b.client.GetObject(context.TODO(), b.config.Bucket, b.getObjectName(file.ID), getOpts)
}

// AddFile implementation for S3 Data Backend
func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (backendDetails string, err error) {
func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (err error) {
putOpts := minio.PutObjectOptions{ContentType: file.Type}

// Configure server side encryption
putOpts.ServerSideEncryption, err = b.getServerSideEncryption(file)
if err != nil {
return err
}

if file.Size > 0 {
_, err = b.client.PutObject(b.config.Bucket, b.getObjectName(file.ID), fileReader, file.Size, minio.PutObjectOptions{ContentType: file.Type})
_, err = b.client.PutObject(context.TODO(), b.config.Bucket, b.getObjectName(file.ID), fileReader, file.Size, putOpts)
} else {
// https://github.com/minio/minio-go/issues/989
// Minio defaults to 128MB chunks and has to actually allocate a buffer of this size before uploading the chunk
// This can lead to very high memory usage when uploading a lot of small files in parallel
// We default to 16MB which allow to store files up to 160GB ( 10000 chunks of 16MB ), feel free to adjust this parameter to your needs.
_, err = b.client.PutObject(b.config.Bucket, b.getObjectName(file.ID), fileReader, -1, minio.PutObjectOptions{ContentType: file.Type, PartSize: b.config.PartSize})
putOpts.PartSize = b.config.PartSize

_, err = b.client.PutObject(context.TODO(), b.config.Bucket, b.getObjectName(file.ID), fileReader, -1, putOpts)
}
return "", err
return err
}

// RemoveFile implementation for S3 Data Backend
func (b *Backend) RemoveFile(file *common.File) (err error) {
return b.client.RemoveObject(b.config.Bucket, b.getObjectName(file.ID))
return b.client.RemoveObject(context.TODO(), b.config.Bucket, b.getObjectName(file.ID), minio.RemoveObjectOptions{})
}

func (b *Backend) getObjectName(name string) string {
Expand Down
83 changes: 83 additions & 0 deletions server/data/s3/sse.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package s3

import (
"encoding/json"
"fmt"

"github.com/minio/minio-go/v7/pkg/encrypt"

"github.com/root-gg/plik/server/common"
)

// Build Server Side Encryption configuration
func (b *Backend) getServerSideEncryption(file *common.File) (sse encrypt.ServerSide, err error) {
switch encrypt.Type(b.config.SSE) {
case "":
return nil, nil
case encrypt.S3:
return encrypt.NewSSE(), nil
case encrypt.SSEC:
key, err := getServerSideEncryptionKey(file)
if err != nil {
return nil, fmt.Errorf("unable to get Server Side Encryption Key : %s", err)
}
return encrypt.NewSSEC([]byte(key))
case encrypt.KMS:
return nil, fmt.Errorf("KMS server side encryption is not yet implemented")
default:
return nil, fmt.Errorf("invalid SSE type %s", b.config.SSE)
}
}

// Generate a 32Bytes / 256bits encryption key
func genServerSideEncryptionKey() string {
return common.GenerateRandomID(32)
}

// Get the SSE Key from the file backend details or generate one and store it in the file backend details
func getServerSideEncryptionKey(file *common.File) (key string, err error) {
// Retrieve the SSE Key from the backend details
if file.BackendDetails != "" {
backendDetails := &BackendDetails{}
err = json.Unmarshal([]byte(file.BackendDetails), backendDetails)
if err != nil {
return "", fmt.Errorf("unable to deserialize backend details : %s", err)
}

if backendDetails.SSEKey != "" {
return backendDetails.SSEKey, nil
}
}

key = genServerSideEncryptionKey()

// Store the SSE Key in the backend details
err = setServerSideEncryptionKey(file, key)
if err != nil {
return "", err
}

return key, nil
}

// Add the SSE Key to the file backend details
func setServerSideEncryptionKey(file *common.File, key string) (err error) {
backendDetails := &BackendDetails{}

if file.BackendDetails != "" {
err = json.Unmarshal([]byte(file.BackendDetails), backendDetails)
if err != nil {
return fmt.Errorf("unable to deserialize backend details : %s", err)
}
}

backendDetails.SSEKey = key

backendDetailsJSON, err := json.Marshal(backendDetails)
if err != nil {
return fmt.Errorf("unable to serialize backend details : %s", err)
}

file.BackendDetails = string(backendDetailsJSON)
return nil
}
4 changes: 2 additions & 2 deletions server/data/stream/stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func (b *Backend) GetFile(file *common.File) (stream io.ReadCloser, err error) {

// AddFile implementation for steam data backend will creates a new steam for the given upload
// and save it on filesystem with the given steam reader
func (b *Backend) AddFile(file *common.File, stream io.Reader) (backendDetails string, err error) {
func (b *Backend) AddFile(file *common.File, stream io.Reader) (err error) {
storeID := file.UploadID + "/" + file.ID

pipeReader, pipeWriter := io.Pipe()
Expand All @@ -61,7 +61,7 @@ func (b *Backend) AddFile(file *common.File, stream io.Reader) (backendDetails s
_, err = io.Copy(pipeWriter, stream)
_ = pipeWriter.Close()

return "", nil
return nil
}

// RemoveFile is not implemented
Expand Down
4 changes: 2 additions & 2 deletions server/data/stream/stream_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ func TestAddGetFile(t *testing.T) {
wg.Add(1)
go func() {
time.Sleep(10 * time.Millisecond)
details, err := backend.AddFile(file, bytes.NewBufferString("data"))
err := backend.AddFile(file, bytes.NewBufferString("data"))
require.NoError(t, err, "unable to add file")
require.NotNil(t, details, "invalid nil details")
require.NotNil(t, file.BackendDetails, "invalid nil details")
wg.Done()
}()

Expand Down
Loading

0 comments on commit 5e192e0

Please sign in to comment.