diff --git a/config.toml.example b/config.toml.example index 6dd6afcd..1b31f198 100644 --- a/config.toml.example +++ b/config.toml.example @@ -26,6 +26,9 @@ path = "test" endpoint_url = "https://s3.us-west-2.amazonaws.com" region = "us-west-2" bucket = "example-bucket-name" + # If you use prefix, you may need to add a path to `server.hostname` setting + # e.g. https://example-bucket-name.s3.us-west-2.amazonaws.com/example/prefix/ + prefix = "example/prefix" # API keys to be used to access Youtube and Vimeo. # These can be either specified as string parameter or array of string (so those will be rotated). diff --git a/pkg/fs/s3.go b/pkg/fs/s3.go index 688d3a0e..16cdb8eb 100644 --- a/pkg/fs/s3.go +++ b/pkg/fs/s3.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "os" + "path" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -24,6 +25,8 @@ type S3Config struct { Region string `toml:"region"` // EndpointURL is an HTTP endpoint of the S3 API EndpointURL string `toml:"endpoint_url"` + // Prefix is a prefix (subfolder) to use to build key names + Prefix string `toml:"prefix"` } // S3 implements file storage for S3-compatible providers. @@ -31,6 +34,7 @@ type S3 struct { api s3iface.S3API uploader *s3manager.Uploader bucket string + prefix string } func NewS3(c S3Config) (*S3, error) { @@ -47,6 +51,7 @@ func NewS3(c S3Config) (*S3, error) { api: s3.New(sess), uploader: s3manager.NewUploader(sess), bucket: c.Bucket, + prefix: c.Prefix, }, nil } @@ -55,21 +60,23 @@ func (s *S3) Open(_name string) (http.File, error) { } func (s *S3) Delete(ctx context.Context, name string) error { + key := s.buildKey(name) _, err := s.api.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{ Bucket: &s.bucket, - Key: &name, + Key: &key, }) return err } func (s *S3) Create(ctx context.Context, name string, reader io.Reader) (int64, error) { - logger := log.WithField("name", name) + key := s.buildKey(name) + logger := log.WithField("key", key) logger.Infof("uploading file to %s", s.bucket) r := &readerWithN{Reader: reader} _, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: &s.bucket, - Key: &name, + Key: &key, Body: r, }) if err != nil { @@ -81,12 +88,13 @@ func (s *S3) Create(ctx context.Context, name string, reader io.Reader) (int64, } func (s *S3) Size(ctx context.Context, name string) (int64, error) { - logger := log.WithField("name", name) + key := s.buildKey(name) + logger := log.WithField("key", key) logger.Debugf("getting file size from %s", s.bucket) resp, err := s.api.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ Bucket: &s.bucket, - Key: &name, + Key: &key, }) if err != nil { if awsErr, ok := err.(awserr.Error); ok { @@ -100,6 +108,10 @@ func (s *S3) Size(ctx context.Context, name string) (int64, error) { return *resp.ContentLength, nil } +func (s *S3) buildKey(name string) string { + return path.Join(s.prefix, name) +} + type readerWithN struct { io.Reader n int diff --git a/pkg/fs/s3_test.go b/pkg/fs/s3_test.go index ecdb65b8..ea403366 100644 --- a/pkg/fs/s3_test.go +++ b/pkg/fs/s3_test.go @@ -18,7 +18,7 @@ import ( func TestS3_Create(t *testing.T) { files := make(map[string][]byte) - stor, err := newMockS3(files) + stor, err := newMockS3(files, "") assert.NoError(t, err) written, err := stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) @@ -32,7 +32,7 @@ func TestS3_Create(t *testing.T) { func TestS3_Size(t *testing.T) { files := make(map[string][]byte) - stor, err := newMockS3(files) + stor, err := newMockS3(files, "") assert.NoError(t, err) _, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) @@ -45,7 +45,7 @@ func TestS3_Size(t *testing.T) { func TestS3_NoSize(t *testing.T) { files := make(map[string][]byte) - stor, err := newMockS3(files) + stor, err := newMockS3(files, "") assert.NoError(t, err) _, err = stor.Size(testCtx, "1/test") @@ -54,7 +54,7 @@ func TestS3_NoSize(t *testing.T) { func TestS3_Delete(t *testing.T) { files := make(map[string][]byte) - stor, err := newMockS3(files) + stor, err := newMockS3(files, "") assert.NoError(t, err) _, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) @@ -70,17 +70,30 @@ func TestS3_Delete(t *testing.T) { assert.False(t, ok) } +func TestS3_BuildKey(t *testing.T) { + files := make(map[string][]byte) + + stor, _ := newMockS3(files, "") + key := stor.buildKey("test-fn") + assert.EqualValues(t, "test-fn", key) + + stor, _ = newMockS3(files, "mock-prefix") + key = stor.buildKey("test-fn") + assert.EqualValues(t, "mock-prefix/test-fn", key) +} + type mockS3API struct { s3iface.S3API files map[string][]byte } -func newMockS3(files map[string][]byte) (*S3, error) { +func newMockS3(files map[string][]byte, prefix string) (*S3, error) { api := &mockS3API{files: files} return &S3{ api: api, uploader: s3manager.NewUploaderWithClient(api), bucket: "mock-bucket", + prefix: prefix, }, nil }