From e521b7d9b2ef8491dc1734a9b017d15a838dca26 Mon Sep 17 00:00:00 2001 From: messikiller Date: Tue, 8 Apr 2025 17:00:15 +0800 Subject: [PATCH 1/2] support aliyun oss [draft] --- go.mod | 14 +- go.sum | 16 + ossfs/file.go | 288 ++++++++++++++++ ossfs/file_info.go | 17 + ossfs/file_test.go | 296 +++++++++++++++++ ossfs/fs.go | 188 +++++++++++ ossfs/fs_test.go | 462 ++++++++++++++++++++++++++ ossfs/fs_utils.go | 36 ++ ossfs/init.go | 14 + ossfs/internal/mocks/FileInfo.go | 139 ++++++++ ossfs/internal/mocks/ObjectManager.go | 293 ++++++++++++++++ ossfs/internal/utils/contract.go | 21 ++ ossfs/internal/utils/init.go | 6 + ossfs/internal/utils/oss.go | 210 ++++++++++++ 14 files changed, 1999 insertions(+), 1 deletion(-) create mode 100644 ossfs/file.go create mode 100644 ossfs/file_info.go create mode 100644 ossfs/file_test.go create mode 100644 ossfs/fs.go create mode 100644 ossfs/fs_test.go create mode 100644 ossfs/fs_utils.go create mode 100644 ossfs/init.go create mode 100644 ossfs/internal/mocks/FileInfo.go create mode 100644 ossfs/internal/mocks/ObjectManager.go create mode 100644 ossfs/internal/utils/contract.go create mode 100644 ossfs/internal/utils/init.go create mode 100644 ossfs/internal/utils/oss.go diff --git a/go.mod b/go.mod index 101c2865..fa24da40 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,16 @@ module github.com/spf13/afero go 1.23.0 -require golang.org/x/text v0.23.0 +require ( + github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1 + github.com/stretchr/testify v1.10.0 + golang.org/x/text v0.23.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + golang.org/x/time v0.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index d00bb390..1862dd3d 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,18 @@ +github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1 h1:sOhpJdR/+lbQniznp3cYSfwQlXbVkT0ccuiZScBrI6Y= +github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= +golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ossfs/file.go b/ossfs/file.go new file mode 100644 index 00000000..630beff8 --- /dev/null +++ b/ossfs/file.go @@ -0,0 +1,288 @@ +package ossfs + +import ( + "errors" + "io" + "os" + "strconv" + "syscall" + + "github.com/spf13/afero" +) + +type File struct { + name string + fs *Fs + openFlag int + offset int64 + fi os.FileInfo + dirty bool + closed bool + isDir bool + preloaded bool + preloadedFd afero.File +} + +func NewOssFile(name string, flag int, fs *Fs) (*File, error) { + return &File{ + name: fs.normFileName(name), + fs: fs, + openFlag: flag, + offset: 0, + dirty: false, + closed: false, + isDir: fs.isDir(fs.normFileName(name)), + preloaded: false, + preloadedFd: nil, + }, nil +} + +func (f *File) preload() error { + pfs := f.fs.preloadFs + if _, err := pfs.Stat(f.name); err == nil { + if e := pfs.Remove(f.name); e != nil { + return e + } + } + pfd, err := f.fs.preloadFs.Create(f.name) + if err != nil { + return err + } + + r, clean, e := f.fs.manager.GetObject(f.fs.ctx, f.fs.bucketName, f.name) + if e != nil { + return e + } + defer clean() + + if _, err := io.Copy(pfd, r); err != nil { + return err + } + + if _, err := pfd.Seek(f.offset, io.SeekStart); err != nil { + return err + } + + f.preloadedFd = pfd + f.preloaded = true + return nil +} + +func (f *File) freshFileInfo() error { + fi, err := f.fs.Stat(f.name) + if err != nil { + return err + } + f.fi = fi + f.dirty = false + return nil +} + +func (f *File) getFileInfo() (os.FileInfo, error) { + if f.dirty { + if err := f.freshFileInfo(); err != nil { + return nil, err + } + } + return f.fi, nil +} + +func (f *File) isReadable() bool { + return !f.closed && (f.openFlag == os.O_RDONLY || f.openFlag == os.O_RDWR) +} + +func (f *File) isWriteable() bool { + return !f.closed && (f.openFlag == os.O_WRONLY || f.openFlag == os.O_RDWR) +} + +func (f *File) isAppendOnly() bool { + return f.isWriteable() && f.openFlag&os.O_APPEND != 0 +} + +func (f *File) Read(p []byte) (int, error) { + if !f.isReadable() || f.isDir { + return 0, syscall.EPERM + } + n, err := f.ReadAt(p, f.offset) + if err != nil { + return 0, err + } + f.offset += int64(n) + return n, err +} + +func (f *File) ReadAt(p []byte, off int64) (int, error) { + if !f.isReadable() || f.isDir { + return 0, syscall.EPERM + } + reader, cleanUp, err := f.fs.manager.GetObjectPart(f.fs.ctx, f.fs.bucketName, f.name, off, off+int64(len(p))) + if err != nil { + return 0, err + } + defer cleanUp() + return reader.Read(p) +} + +func (f *File) Seek(offset int64, whence int) (int64, error) { + if (!f.isReadable() && !f.isWriteable()) || f.isDir { + return 0, syscall.EPERM + } + fi, err := f.getFileInfo() + if err != nil { + return 0, err + } + max := fi.Size() + var newOffset int64 + switch whence { + case io.SeekCurrent: + newOffset = f.offset + offset + case io.SeekStart: + newOffset = offset + case io.SeekEnd: + newOffset = max + offset + default: + return 0, errors.New("Invalid whence value: " + strconv.Itoa(whence)) + } + if newOffset < 0 || newOffset > max { + return 0, afero.ErrOutOfRange + } + f.offset = newOffset + return f.offset, nil +} + +func (f *File) doAppend(p []byte) (int, error) { + if !f.isWriteable() { + return 0, syscall.EPERM + } + fi, err := f.getFileInfo() + if err != nil { + return 0, err + } + return f.doWriteAt(p, fi.Size()) +} + +func (f *File) Write(p []byte) (int, error) { + if !f.isWriteable() { + return 0, syscall.EPERM + } + if f.isAppendOnly() { + return f.doAppend(p) + } + n, e := f.doWriteAt(p, f.offset) + if e != nil { + return 0, e + } + f.offset += int64(n) + return n, e +} + +func (f *File) doWriteAt(p []byte, off int64) (int, error) { + if f.isDir { + return 0, syscall.EPERM + } + + if !f.preloaded { + if err := f.preload(); err != nil { + return 0, err + } + } + + n, e := f.preloadedFd.WriteAt(p, off) + f.dirty = true + if f.fs.autoSync { + f.Sync() + } + return n, e +} + +func (f *File) WriteAt(p []byte, off int64) (int, error) { + if !f.isWriteable() || f.isAppendOnly() { + return 0, syscall.EPERM + } + return f.doWriteAt(p, off) +} + +func (f *File) Close() error { + f.Sync() + f.closed = true + delete(f.fs.openedFiles, f.name) + if f.preloaded { + err := f.fs.preloadFs.Remove(f.name) + if err != nil { + return err + } + err = f.preloadedFd.Close() + if err != nil { + return err + } + f.preloadedFd = nil + f.preloaded = false + } + return nil +} + +func (f *File) Name() string { + return f.name +} + +func (f *File) Readdir(count int) ([]os.FileInfo, error) { + if !f.isReadable() { + return nil, syscall.EPERM + } + + fis, err := f.fs.manager.ListObjects(f.fs.ctx, f.fs.bucketName, f.fs.ensureAsDir(f.name), count) + return fis, err +} + +func (f *File) Readdirnames(n int) ([]string, error) { + if !f.isReadable() { + return nil, syscall.EPERM + } + + fis, err := f.Readdir(n) + if err != nil { + return nil, err + } + var fNames []string + for _, fi := range fis { + fNames = append(fNames, fi.Name()) + } + + return fNames, nil +} + +func (f *File) Stat() (os.FileInfo, error) { + if f.dirty { + err := f.freshFileInfo() + if err != nil { + return nil, err + } + } + return f.fi, nil +} + +func (f *File) Sync() error { + if f.preloaded { + if _, err := f.fs.manager.PutObject(f.fs.ctx, f.fs.bucketName, f.name, f.preloadedFd); err != nil { + return err + } + } + if f.dirty { + if err := f.freshFileInfo(); err != nil { + return err + } + } + return nil +} + +func (f *File) Truncate(size int64) error { + if !f.isWriteable() || f.isDir { + return syscall.EPERM + } + _, err := f.WriteAt([]byte(""), 0) + return err +} + +func (f *File) WriteString(s string) (int, error) { + return f.Write([]byte(s)) +} diff --git a/ossfs/file_info.go b/ossfs/file_info.go new file mode 100644 index 00000000..ea31d2da --- /dev/null +++ b/ossfs/file_info.go @@ -0,0 +1,17 @@ +package ossfs + +import ( + "time" + + "github.com/spf13/afero/ossfs/internal/utils" +) + +type FileInfo struct { + *utils.OssObjectMeta +} + +func NewFileInfo(name string, size int64, updatedAt time.Time) *FileInfo { + return &FileInfo{ + OssObjectMeta: utils.NewOssObjectMeta(name, size, updatedAt), + } +} diff --git a/ossfs/file_test.go b/ossfs/file_test.go new file mode 100644 index 00000000..fba21bd2 --- /dev/null +++ b/ossfs/file_test.go @@ -0,0 +1,296 @@ +package ossfs + +import ( + "io" + "os" + "strings" + "syscall" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/spf13/afero/ossfs/internal/mocks" + "github.com/spf13/afero/ossfs/internal/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func getMockedFs() *Fs { + fs := NewOssFs("test-ak", "test-sk", "test-region", "test-bucket") + fs.manager = &mocks.ObjectManager{} + return fs +} + +func getMockedFile(name string, flag int, fs *Fs) *File { + f, _ := NewOssFile(name, flag, fs) + return f +} + +func TestNewOssFile(t *testing.T) { + t.Run("create new file with read flag", func(t *testing.T) { + fs := &Fs{} + file, err := NewOssFile("testfile", os.O_RDONLY, fs) + assert.NoError(t, err) + assert.Equal(t, "testfile", file.name) + assert.Equal(t, os.O_RDONLY, file.openFlag) + assert.Equal(t, fs, file.fs) + assert.False(t, file.dirty) + assert.False(t, file.closed) + assert.False(t, file.isDir) + assert.False(t, file.preloaded) + assert.Nil(t, file.preloadedFd) + }) + + t.Run("create new file with write flag", func(t *testing.T) { + fs := &Fs{} + file, err := NewOssFile("testfile", os.O_WRONLY, fs) + assert.NoError(t, err) + assert.Equal(t, "testfile", file.name) + assert.Equal(t, os.O_WRONLY, file.openFlag) + assert.Equal(t, fs, file.fs) + assert.False(t, file.dirty) + assert.False(t, file.closed) + assert.False(t, file.isDir) + assert.False(t, file.preloaded) + assert.Nil(t, file.preloadedFd) + }) + + t.Run("create new directory", func(t *testing.T) { + fs := &Fs{} + file, err := NewOssFile("testdir/", os.O_RDONLY, fs) + assert.NoError(t, err) + assert.Equal(t, "testdir/", file.name) + assert.Equal(t, os.O_RDONLY, file.openFlag) + assert.Equal(t, fs, file.fs) + assert.False(t, file.dirty) + assert.False(t, file.closed) + assert.True(t, file.isDir) + assert.False(t, file.preloaded) + assert.Nil(t, file.preloadedFd) + }) + + t.Run("normalize file name", func(t *testing.T) { + fs := &Fs{} + file, err := NewOssFile("/path/testfile", os.O_RDONLY, fs) + assert.NoError(t, err) + assert.Equal(t, "path/testfile", file.name) + }) +} + +func TestRead(t *testing.T) { + t.Run("Read with unreadable flag return error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_WRONLY, fs) + + p := make([]byte, 0) + _, e := f.Read(p) + + assert.Error(t, e) + assert.NotNil(t, e) + }) + + t.Run("Read on directory return error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testdir", os.O_RDONLY, fs) + f.isDir = true + + p := make([]byte, 10) + _, e := f.Read(p) + + assert.Error(t, e) + assert.Equal(t, syscall.EPERM, e) + }) + + t.Run("Read on closed file return error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDONLY, fs) + f.closed = true + + p := make([]byte, 10) + _, e := f.Read(p) + + assert.Error(t, e) + assert.Equal(t, syscall.EPERM, e) + }) + + t.Run("Successful read updates offset", func(t *testing.T) { + fs := getMockedFs() + var cu utils.CleanUp = func() {} + mockManager := fs.manager.(*mocks.ObjectManager) + mockManager.On("GetObjectPart", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + // Return(&mockReadCloser{data: []byte("testdata")}, cu, nil) + Return(strings.NewReader("testdata"), cu, nil) + + f := getMockedFile("testfile", os.O_RDONLY, fs) + p := make([]byte, 8) + n, err := f.Read(p) + + assert.NoError(t, err) + assert.Equal(t, 8, n) + assert.Equal(t, int64(8), f.offset) + }) + + t.Run("ReadAt error propagates", func(t *testing.T) { + fs := getMockedFs() + mockManager := fs.manager.(*mocks.ObjectManager) + mockManager.On("GetObjectPart", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(nil, nil, syscall.EIO) + + f := getMockedFile("testfile", os.O_RDONLY, fs) + p := make([]byte, 8) + _, err := f.Read(p) + + assert.Error(t, err) + assert.Equal(t, syscall.EIO, err) + }) +} + +func TestReadAt(t *testing.T) { + t.Run("ReadAt with unreadable flag return error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_WRONLY, fs) + + p := make([]byte, 0) + _, e := f.ReadAt(p, 0) + + assert.Error(t, e) + assert.NotNil(t, e) + }) + + t.Run("ReadAt on dir return error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("/path/to/dir/", os.O_WRONLY, fs) + + p := make([]byte, 0) + _, e := f.ReadAt(p, 0) + + assert.Error(t, e) + assert.NotNil(t, e) + }) + + t.Run("ReadAt success", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDONLY, fs) + + p := make([]byte, 4) + + var cu utils.CleanUp = func() {} + off := int64(5) + m := &mocks.ObjectManager{} + m. + On("GetObjectPart", f.fs.ctx, f.fs.bucketName, f.name, off, off+int64(len(p))). + Return(strings.NewReader("test result"), cu, nil) + fs.manager = m + + n, e := f.ReadAt(p, off) + + assert.Nil(t, e) + assert.Equal(t, 4, n) + assert.Equal(t, "test", string(p)) + }) +} + +func TestSeek(t *testing.T) { + t.Run("Seek on unreadable/unwritable file returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_WRONLY, fs) + f.closed = true + + _, err := f.Seek(0, io.SeekStart) + assert.Error(t, err) + assert.Equal(t, syscall.EPERM, err) + }) + + t.Run("Seek on directory returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testdir", os.O_RDONLY, fs) + f.isDir = true + + _, err := f.Seek(0, io.SeekStart) + assert.Error(t, err) + assert.Equal(t, syscall.EPERM, err) + }) + + t.Run("SeekStart sets correct offset", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + + offset, err := f.Seek(10, io.SeekStart) + assert.NoError(t, err) + assert.Equal(t, int64(10), offset) + assert.Equal(t, int64(10), f.offset) + }) + + t.Run("SeekCurrent adjusts offset correctly", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + f.offset = 5 + + offset, err := f.Seek(5, io.SeekCurrent) + assert.NoError(t, err) + assert.Equal(t, int64(10), offset) + assert.Equal(t, int64(10), f.offset) + }) + + t.Run("SeekEnd adjusts offset correctly", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + + offset, err := f.Seek(-10, io.SeekEnd) + assert.NoError(t, err) + assert.Equal(t, int64(90), offset) + assert.Equal(t, int64(90), f.offset) + }) + + t.Run("Seek beyond file size returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + + _, err := f.Seek(101, io.SeekStart) + assert.Error(t, err) + assert.Equal(t, afero.ErrOutOfRange, err) + }) + + t.Run("Seek negative offset returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + + _, err := f.Seek(-1, io.SeekStart) + assert.Error(t, err) + assert.Equal(t, afero.ErrOutOfRange, err) + }) + + t.Run("Seek with invalid whence returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDWR, fs) + f.fi = NewFileInfo("testfile", 100, time.Now()) + + _, err := f.Seek(0, 3) + assert.Error(t, err) + }) +} + +func TestWrite(t *testing.T) { + t.Run("Write unwritable file returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("testfile", os.O_RDONLY, fs) + _, e := f.Write([]byte("test input string")) + + assert.Error(t, e) + assert.NotNil(t, e) + }) + + t.Run("Write dir returns error", func(t *testing.T) { + fs := getMockedFs() + f := getMockedFile("/path/to/test_dir/", os.O_WRONLY, fs) + _, e := f.Write([]byte("test input string")) + + assert.Error(t, e) + assert.NotNil(t, e) + }) +} diff --git a/ossfs/fs.go b/ossfs/fs.go new file mode 100644 index 00000000..05ef2ffd --- /dev/null +++ b/ossfs/fs.go @@ -0,0 +1,188 @@ +package ossfs + +import ( + "context" + "errors" + "os" + "strings" + "time" + + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials" + "github.com/spf13/afero" + "github.com/spf13/afero/ossfs/internal/utils" +) + +const ( + defaultFileMode = 0o755 + defaultFileFlag = os.O_RDWR +) + +type Fs struct { + manager utils.ObjectManager + bucketName string + separator string + autoSync bool + openedFiles map[string]afero.File + preloadFs afero.Fs + ctx context.Context +} + +func NewOssFs(accessKeyId, accessKeySecret, region, bucket string) *Fs { + ossCfg := oss.LoadDefaultConfig(). + WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyId, accessKeySecret)). + WithRegion(region) + + return &Fs{ + manager: &utils.OssObjectManager{ + Client: oss.NewClient(ossCfg), + }, + bucketName: bucket, + separator: "/", + autoSync: true, + openedFiles: make(map[string]afero.File), + preloadFs: afero.NewMemMapFs(), + ctx: context.Background(), + } +} + +func (fs *Fs) WithContext(ctx context.Context) *Fs { + fs.ctx = ctx + return fs +} + +// Create creates a new empty file and open it, return the open file and error +// if any happens. +func (fs *Fs) Create(name string) (afero.File, error) { + n := fs.normFileName(name) + r := strings.NewReader("") + if _, err := fs.manager.PutObject(fs.ctx, fs.bucketName, n, r); err != nil { + return nil, err + } + return NewOssFile(n, defaultFileFlag, fs) +} + +// Mkdir creates a directory in the filesystem, return an error if any +// happens. +func (fs *Fs) Mkdir(name string, perm os.FileMode) error { + return fs.MkdirAll(fs.ensureAsDir(name), perm) +} + +// MkdirAll creates a directory path and all parents that does not exist +// yet. +func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { + dirName := fs.ensureAsDir(path) + r := strings.NewReader("") + _, err := fs.manager.PutObject(fs.ctx, fs.bucketName, dirName, r) + return err +} + +// Open opens a file, returning it or an error, if any happens. +func (fs *Fs) Open(name string) (afero.File, error) { + return fs.OpenFile(name, defaultFileFlag, defaultFileMode) +} + +// OpenFile opens a file using the given flags and the given mode. +func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + name = fs.normFileName(name) + file, found := fs.openedFiles[name] + if found && file.(*File).openFlag == flag { + return file, nil + } + + f, err := NewOssFile(name, flag, fs) + if err != nil { + return nil, err + } + + existed := false + existed, err = fs.manager.IsObjectExist(fs.ctx, fs.bucketName, name) + if err != nil { + return nil, err + } + + if !existed && f.openFlag&os.O_CREATE == 0 { + return nil, afero.ErrFileNotFound + } + + if !existed && f.openFlag*os.O_CREATE != 0 { + if _, err := fs.Create(f.name); err != nil { + return nil, err + } + } + + if f.openFlag&os.O_TRUNC != 0 { + _, err := f.fs.manager.PutObject(fs.ctx, fs.bucketName, f.name, strings.NewReader("")) + if err != nil { + return nil, err + } + } + + fs.openedFiles[name] = f + + return f, nil +} + +// Remove removes a file identified by name, returning an error, if any +// happens. +func (fs *Fs) Remove(name string) error { + return fs.manager.DeleteObject(fs.ctx, fs.bucketName, fs.normFileName(name)) +} + +// RemoveAll removes a directory path and any children it contains. It +// does not fail if the path does not exist (return nil). +func (fs *Fs) RemoveAll(path string) error { + dir := fs.ensureAsDir(path) + fis, err := fs.manager.ListAllObjects(fs.ctx, fs.bucketName, dir) + if err != nil { + return err + } + for _, fi := range fis { + err = fs.manager.DeleteObject(fs.ctx, fs.bucketName, fi.Name()) + if err != nil { + return err + } + } + return nil +} + +// Rename renames a file. +func (fs *Fs) Rename(oldname, newname string) error { + err := fs.manager.CopyObject(fs.ctx, fs.bucketName, oldname, newname) + if err != nil { + return err + } + err = fs.manager.DeleteObject(fs.ctx, fs.bucketName, oldname) + return err +} + +// Stat returns a FileInfo describing the named file, or an error, if any +// happens. +func (fs *Fs) Stat(name string) (os.FileInfo, error) { + fi, err := fs.manager.GetObjectMeta(fs.ctx, fs.bucketName, fs.normFileName(name)) + if err != nil { + return nil, err + } + + return fi, err +} + +// The name of this FileSystem +func (fs *Fs) Name() string { + return "OssFs" +} + +// Chmod changes the mode of the named file to mode. +func (fs *Fs) Chmod(name string, mode os.FileMode) error { + return errors.New("OSS: method Chmod is not implemented") +} + +// Chown changes the uid and gid of the named file. +func (fs *Fs) Chown(name string, uid, gid int) error { + return errors.New("OSS: method Chown is not implemented") +} + +// Chtimes changes the access and modification times of the named file +func (fs *Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return errors.New("OSS: method Chtimes is not implemented") +} diff --git a/ossfs/fs_test.go b/ossfs/fs_test.go new file mode 100644 index 00000000..8d09ecf8 --- /dev/null +++ b/ossfs/fs_test.go @@ -0,0 +1,462 @@ +package ossfs + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/spf13/afero/ossfs/internal/mocks" + "github.com/stretchr/testify/assert" +) + +func TestNewOssFs(t *testing.T) { + tests := []struct { + name string + accessKeyId string + accessKeySecret string + region string + bucket string + expected *Fs + }{ + { + name: "valid credentials", + accessKeyId: "testKeyId", + accessKeySecret: "testKeySecret", + region: "test-region", + bucket: "test-bucket", + expected: &Fs{ + bucketName: "test-bucket", + separator: "/", + autoSync: true, + openedFiles: make(map[string]afero.File), + preloadFs: afero.NewMemMapFs(), + ctx: context.Background(), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewOssFs(tt.accessKeyId, tt.accessKeySecret, tt.region, tt.bucket) + assert.NotNil(t, got.manager) + assert.Equal(t, tt.expected.bucketName, got.bucketName) + assert.Equal(t, tt.expected.separator, got.separator) + assert.Equal(t, tt.expected.autoSync, got.autoSync) + assert.NotNil(t, got.openedFiles) + assert.NotNil(t, got.preloadFs) + assert.NotNil(t, got.ctx) + }) + } +} + +func TestFsWithContext(t *testing.T) { + type bgMeta string + tests := []struct { + name string + fs *Fs + ctx context.Context + expected *Fs + }{ + { + name: "set new context", + fs: &Fs{ + ctx: context.Background(), + }, + ctx: context.WithValue(context.Background(), bgMeta("testKey"), bgMeta("testValue")), + expected: &Fs{ + ctx: context.WithValue(context.Background(), bgMeta("testKey"), bgMeta("testValue")), + }, + }, + { + name: "set nil context", + fs: &Fs{ + ctx: context.Background(), + }, + ctx: nil, + expected: &Fs{ + ctx: nil, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.fs.WithContext(tt.ctx) + assert.Equal(t, tt.expected.ctx, got.ctx) + assert.Equal(t, tt.fs, got) + }) + } +} + +func TestFsCreate(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("create simple success", func(t *testing.T) { + m.On("PutObject", fs.ctx, bucket, "test.txt", strings.NewReader("")).Return(true, nil).Once() + file, err := fs.Create("test.txt") + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + m.AssertExpectations(t) + }) + + t.Run("create prefixed file path success", func(t *testing.T) { + m. + On("PutObject", fs.ctx, bucket, "path/to/test.txt", strings.NewReader("")). + Return(true, nil). + Once() + file, err := fs.Create("/path/to/test.txt") + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + m.AssertExpectations(t) + }) + + t.Run("create dir path success", func(t *testing.T) { + m. + On("PutObject", ctx, bucket, "path/to/test_dir/", strings.NewReader("")). + Return(true, nil). + Once() + file, err := fs.Create("/path/to/test_dir/") + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + assert.Equal(t, "path/to/test_dir/", file.Name()) + m.AssertExpectations(t) + }) + + t.Run("create failure", func(t *testing.T) { + m.On("PutObject", fs.ctx, bucket, "test2.txt", strings.NewReader("")).Return(false, afero.ErrFileNotFound).Once() + _, err := fs.Create("test2.txt") + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) +} + +func TestFsMkdirAll(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("MkDirAll simple success", func(t *testing.T) { + m. + On("PutObject", fs.ctx, bucket, "path/to/test_dir/", strings.NewReader("")). + Return(true, nil). + Once() + err := fs.MkdirAll("/path/to/test_dir/", defaultFileMode) + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("MkDirAll failure", func(t *testing.T) { + m. + On("PutObject", fs.ctx, bucket, "path/to/test_dir/", strings.NewReader("")). + Return(false, afero.ErrFileClosed). + Once() + err := fs.MkdirAll("/path/to/test_dir/", defaultFileMode) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileClosed) + m.AssertExpectations(t) + }) +} + +func TestFsOpenFile(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + openedFiles: make(map[string]afero.File), + } + + t.Run("open existing file success", func(t *testing.T) { + m.On("IsObjectExist", ctx, bucket, "test.txt").Return(true, nil).Once() + file, err := fs.OpenFile("test.txt", os.O_RDONLY, 0644) + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + m.AssertExpectations(t) + }) + + t.Run("open non-existing file with create flag success", func(t *testing.T) { + m.On("IsObjectExist", ctx, bucket, "new.txt").Return(false, nil).Once() + m.On("PutObject", ctx, bucket, "new.txt", strings.NewReader("")).Return(true, nil).Once() + file, err := fs.OpenFile("new.txt", os.O_CREATE|os.O_RDWR, 0644) + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + m.AssertExpectations(t) + }) + + t.Run("open file with truncate flag success", func(t *testing.T) { + m.On("IsObjectExist", ctx, bucket, "trunc.txt").Return(true, nil).Once() + m.On("PutObject", ctx, bucket, "trunc.txt", strings.NewReader("")).Return(true, nil).Once() + file, err := fs.OpenFile("trunc.txt", os.O_TRUNC|os.O_RDWR, 0644) + assert.Nil(t, err) + assert.NotNil(t, file) + assert.Implements(t, (*afero.File)(nil), file) + m.AssertExpectations(t) + }) + + t.Run("open existing file from cache", func(t *testing.T) { + cachedFile := &File{name: "cached.txt", openFlag: os.O_RDONLY} + fs.openedFiles["cached.txt"] = cachedFile + file, err := fs.OpenFile("cached.txt", os.O_RDONLY, 0644) + assert.Nil(t, err) + assert.Equal(t, cachedFile, file) + assert.Implements(t, (*afero.File)(nil), file) + }) + + t.Run("open non-existing file without create flag fails", func(t *testing.T) { + m.On("IsObjectExist", ctx, bucket, "nonexist.txt").Return(false, nil).Once() + _, err := fs.OpenFile("nonexist.txt", os.O_RDONLY, 0644) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) + + t.Run("open file with check exist error fails", func(t *testing.T) { + m.On("IsObjectExist", ctx, bucket, "error.txt").Return(false, afero.ErrFileNotFound).Once() + _, err := fs.OpenFile("error.txt", os.O_RDONLY, 0644) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) +} + +func TestFsRemove(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("remove file success", func(t *testing.T) { + m.On("DeleteObject", fs.ctx, bucket, "test.txt").Return(nil).Once() + err := fs.Remove("test.txt") + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("remove prefixed file success", func(t *testing.T) { + m.On("DeleteObject", fs.ctx, bucket, "path/to/test.txt").Return(nil).Once() + err := fs.Remove("/path/to/test.txt") + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("remove non-existent file", func(t *testing.T) { + m.On("DeleteObject", fs.ctx, bucket, "nonexistent.txt").Return(afero.ErrFileNotFound).Once() + err := fs.Remove("nonexistent.txt") + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) +} + +func TestFsRemoveAll(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("remove non-empty directory", func(t *testing.T) { + dirPath := "path/to/dir/" + files := []os.FileInfo{ + NewFileInfo("path/to/dir/file1.txt", 100, time.Now()), + NewFileInfo("path/to/dir/file2.txt", 200, time.Now()), + NewFileInfo("path/to/dir/subdir/", 0, time.Now()), + } + + m.On("ListAllObjects", ctx, bucket, dirPath).Return(files, nil).Once() + m.On("DeleteObject", ctx, bucket, "path/to/dir/file1.txt").Return(nil).Once() + m.On("DeleteObject", ctx, bucket, "path/to/dir/file2.txt").Return(nil).Once() + m.On("DeleteObject", ctx, bucket, "path/to/dir/subdir/").Return(nil).Once() + + err := fs.RemoveAll(dirPath) + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("remove empty directory", func(t *testing.T) { + dirPath := "empty/dir/" + m.On("ListAllObjects", ctx, bucket, dirPath).Return([]os.FileInfo{}, nil).Once() + + err := fs.RemoveAll(dirPath) + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("remove non-existent path", func(t *testing.T) { + nonExistentPath := "nonexistent/path/" + m.On("ListAllObjects", ctx, bucket, nonExistentPath).Return([]os.FileInfo{}, nil).Once() + + err := fs.RemoveAll(nonExistentPath) + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("list objects failure", func(t *testing.T) { + dirPath := "path/to/dir/" + m.On("ListAllObjects", ctx, bucket, dirPath).Return(nil, afero.ErrFileNotFound).Once() + + err := fs.RemoveAll(dirPath) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) + + t.Run("delete object failure", func(t *testing.T) { + dirPath := "path/to/dir/" + files := []os.FileInfo{ + NewFileInfo("path/to/dir/file1.txt", 0, time.Now()), + } + + m.On("ListAllObjects", ctx, bucket, dirPath).Return(files, nil).Once() + m.On("DeleteObject", ctx, bucket, "path/to/dir/file1.txt").Return(afero.ErrFileNotFound).Once() + + err := fs.RemoveAll(dirPath) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) +} + +func TestFsRename(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("successful rename", func(t *testing.T) { + oldname := "old/file.txt" + newname := "new/file.txt" + + m.On("CopyObject", ctx, bucket, oldname, newname).Return(nil).Once() + m.On("DeleteObject", ctx, bucket, oldname).Return(nil).Once() + + err := fs.Rename(oldname, newname) + assert.Nil(t, err) + m.AssertExpectations(t) + }) + + t.Run("copy failure", func(t *testing.T) { + oldname := "old/file.txt" + newname := "new/file.txt" + + m.On("CopyObject", ctx, bucket, oldname, newname).Return(afero.ErrFileNotFound).Once() + + err := fs.Rename(oldname, newname) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) + + t.Run("delete failure after successful copy", func(t *testing.T) { + oldname := "old/file.txt" + newname := "new/file.txt" + + m.On("CopyObject", ctx, bucket, oldname, newname).Return(nil).Once() + m.On("DeleteObject", ctx, bucket, oldname).Return(afero.ErrFileNotFound).Once() + + err := fs.Rename(oldname, newname) + assert.NotNil(t, err) + assert.ErrorIs(t, err, afero.ErrFileNotFound) + m.AssertExpectations(t) + }) +} + +func TestFsStat(t *testing.T) { + m := &mocks.ObjectManager{} + bucket := "test-bucket" + ctx := context.TODO() + + fs := &Fs{ + manager: m, + bucketName: bucket, + ctx: ctx, + separator: "/", + } + + t.Run("stat file success", func(t *testing.T) { + expectedInfo := &mocks.FileInfo{} + m.On("GetObjectMeta", fs.ctx, bucket, "test.txt").Return(expectedInfo, nil).Once() + info, err := fs.Stat("test.txt") + assert.Nil(t, err) + assert.Equal(t, expectedInfo, info) + m.AssertExpectations(t) + }) + + t.Run("stat prefixed file path success", func(t *testing.T) { + expectedInfo := &mocks.FileInfo{} + m.On("GetObjectMeta", fs.ctx, bucket, "path/to/test.txt").Return(expectedInfo, nil).Once() + info, err := fs.Stat("/path/to/test.txt") + assert.Nil(t, err) + assert.Equal(t, expectedInfo, info) + m.AssertExpectations(t) + }) + + t.Run("stat dir path success", func(t *testing.T) { + expectedInfo := &mocks.FileInfo{} + m.On("GetObjectMeta", fs.ctx, bucket, "path/to/dir/").Return(expectedInfo, nil).Once() + info, err := fs.Stat("/path/to/dir/") + assert.Nil(t, err) + assert.Equal(t, expectedInfo, info) + m.AssertExpectations(t) + }) + + t.Run("stat non-existent file", func(t *testing.T) { + m.On("GetObjectMeta", fs.ctx, bucket, "nonexistent.txt").Return(nil, os.ErrNotExist).Once() + _, err := fs.Stat("nonexistent.txt") + assert.NotNil(t, err) + assert.ErrorIs(t, err, os.ErrNotExist) + m.AssertExpectations(t) + }) +} + +func TestFsName(t *testing.T) { + fs := &Fs{} + name := fs.Name() + assert.Equal(t, "OssFs", name) +} diff --git a/ossfs/fs_utils.go b/ossfs/fs_utils.go new file mode 100644 index 00000000..2a14609c --- /dev/null +++ b/ossfs/fs_utils.go @@ -0,0 +1,36 @@ +package ossfs + +import ( + "strings" +) + +func (fs *Fs) isDir(s string) bool { + sep := fs.separator + if fs.separator == "" { + sep = "/" + } + return strings.HasSuffix(s, sep) +} + +func (fs *Fs) ensureAsDir(s string) string { + sep := fs.separator + if fs.separator == "" { + sep = "/" + } + s = fs.normFileName(s) + if !strings.HasSuffix(s, sep) { + s = s + sep + } + return s +} + +func (fs *Fs) normFileName(s string) string { + sep := fs.separator + if fs.separator == "" { + sep = "/" + } + s = strings.TrimLeft(s, "/\\") + s = strings.Replace(s, "\\", sep, -1) + s = strings.Replace(s, "/", sep, -1) + return s +} diff --git a/ossfs/init.go b/ossfs/init.go new file mode 100644 index 00000000..3a8ae873 --- /dev/null +++ b/ossfs/init.go @@ -0,0 +1,14 @@ +package ossfs + +import ( + "os" + + "github.com/spf13/afero" +) + +func init() { + // Ensure oss.Fs implements afero.Fs interface + var _ afero.Fs = (*Fs)(nil) + var _ afero.File = (*File)(nil) + var _ os.FileInfo = (*FileInfo)(nil) +} diff --git a/ossfs/internal/mocks/FileInfo.go b/ossfs/internal/mocks/FileInfo.go new file mode 100644 index 00000000..64328f2b --- /dev/null +++ b/ossfs/internal/mocks/FileInfo.go @@ -0,0 +1,139 @@ +// Code generated by mockery v2.53.3. DO NOT EDIT. + +package mocks + +import ( + "os" + time "time" + + mock "github.com/stretchr/testify/mock" +) + +// FileInfo is an autogenerated mock type for the FileInfo type +type FileInfo struct { + mock.Mock +} + +// IsDir provides a mock function with no fields +func (_m *FileInfo) IsDir() bool { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for IsDir") + } + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// ModTime provides a mock function with no fields +func (_m *FileInfo) ModTime() time.Time { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ModTime") + } + + var r0 time.Time + if rf, ok := ret.Get(0).(func() time.Time); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Time) + } + + return r0 +} + +// Mode provides a mock function with no fields +func (_m *FileInfo) Mode() os.FileMode { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Mode") + } + + var r0 os.FileMode + if rf, ok := ret.Get(0).(func() os.FileMode); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(os.FileMode) + } + + return r0 +} + +// Name provides a mock function with no fields +func (_m *FileInfo) Name() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Name") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// Size provides a mock function with no fields +func (_m *FileInfo) Size() int64 { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Size") + } + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + return r0 +} + +// Sys provides a mock function with no fields +func (_m *FileInfo) Sys() interface{} { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Sys") + } + + var r0 interface{} + if rf, ok := ret.Get(0).(func() interface{}); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interface{}) + } + } + + return r0 +} + +// NewFileInfo creates a new instance of FileInfo. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewFileInfo(t interface { + mock.TestingT + Cleanup(func()) +}) *FileInfo { + mock := &FileInfo{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/ossfs/internal/mocks/ObjectManager.go b/ossfs/internal/mocks/ObjectManager.go new file mode 100644 index 00000000..bb3e087b --- /dev/null +++ b/ossfs/internal/mocks/ObjectManager.go @@ -0,0 +1,293 @@ +// Code generated by mockery v2.53.3. DO NOT EDIT. + +package mocks + +import ( + context "context" + fs "io/fs" + + io "io" + + mock "github.com/stretchr/testify/mock" + + utils "github.com/spf13/afero/ossfs/internal/utils" +) + +// ObjectManager is an autogenerated mock type for the ObjectManager type +type ObjectManager struct { + mock.Mock +} + +// CopyObject provides a mock function with given fields: ctx, bucket, srcName, targetName +func (_m *ObjectManager) CopyObject(ctx context.Context, bucket string, srcName string, targetName string) error { + ret := _m.Called(ctx, bucket, srcName, targetName) + + if len(ret) == 0 { + panic("no return value specified for CopyObject") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string) error); ok { + r0 = rf(ctx, bucket, srcName, targetName) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// DeleteObject provides a mock function with given fields: ctx, bucket, name +func (_m *ObjectManager) DeleteObject(ctx context.Context, bucket string, name string) error { + ret := _m.Called(ctx, bucket, name) + + if len(ret) == 0 { + panic("no return value specified for DeleteObject") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, bucket, name) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetObject provides a mock function with given fields: ctx, bucket, name +func (_m *ObjectManager) GetObject(ctx context.Context, bucket string, name string) (io.Reader, utils.CleanUp, error) { + ret := _m.Called(ctx, bucket, name) + + if len(ret) == 0 { + panic("no return value specified for GetObject") + } + + var r0 io.Reader + var r1 utils.CleanUp + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (io.Reader, utils.CleanUp, error)); ok { + return rf(ctx, bucket, name) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) io.Reader); ok { + r0 = rf(ctx, bucket, name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.Reader) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) utils.CleanUp); ok { + r1 = rf(ctx, bucket, name) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(utils.CleanUp) + } + } + + if rf, ok := ret.Get(2).(func(context.Context, string, string) error); ok { + r2 = rf(ctx, bucket, name) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// GetObjectMeta provides a mock function with given fields: ctx, bucket, name +func (_m *ObjectManager) GetObjectMeta(ctx context.Context, bucket string, name string) (fs.FileInfo, error) { + ret := _m.Called(ctx, bucket, name) + + if len(ret) == 0 { + panic("no return value specified for GetObjectMeta") + } + + var r0 fs.FileInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (fs.FileInfo, error)); ok { + return rf(ctx, bucket, name) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) fs.FileInfo); ok { + r0 = rf(ctx, bucket, name) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(fs.FileInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, bucket, name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetObjectPart provides a mock function with given fields: ctx, bucket, name, start, end +func (_m *ObjectManager) GetObjectPart(ctx context.Context, bucket string, name string, start int64, end int64) (io.Reader, utils.CleanUp, error) { + ret := _m.Called(ctx, bucket, name, start, end) + + if len(ret) == 0 { + panic("no return value specified for GetObjectPart") + } + + var r0 io.Reader + var r1 utils.CleanUp + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64) (io.Reader, utils.CleanUp, error)); ok { + return rf(ctx, bucket, name, start, end) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64) io.Reader); ok { + r0 = rf(ctx, bucket, name, start, end) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.Reader) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, int64, int64) utils.CleanUp); ok { + r1 = rf(ctx, bucket, name, start, end) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(utils.CleanUp) + } + } + + if rf, ok := ret.Get(2).(func(context.Context, string, string, int64, int64) error); ok { + r2 = rf(ctx, bucket, name, start, end) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// IsObjectExist provides a mock function with given fields: ctx, bucket, name +func (_m *ObjectManager) IsObjectExist(ctx context.Context, bucket string, name string) (bool, error) { + ret := _m.Called(ctx, bucket, name) + + if len(ret) == 0 { + panic("no return value specified for IsObjectExist") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (bool, error)); ok { + return rf(ctx, bucket, name) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) bool); ok { + r0 = rf(ctx, bucket, name) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, bucket, name) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListAllObjects provides a mock function with given fields: ctx, bucket, prefix +func (_m *ObjectManager) ListAllObjects(ctx context.Context, bucket string, prefix string) ([]fs.FileInfo, error) { + ret := _m.Called(ctx, bucket, prefix) + + if len(ret) == 0 { + panic("no return value specified for ListAllObjects") + } + + var r0 []fs.FileInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]fs.FileInfo, error)); ok { + return rf(ctx, bucket, prefix) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) []fs.FileInfo); ok { + r0 = rf(ctx, bucket, prefix) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]fs.FileInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, bucket, prefix) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ListObjects provides a mock function with given fields: ctx, bucket, prefix, count +func (_m *ObjectManager) ListObjects(ctx context.Context, bucket string, prefix string, count int) ([]fs.FileInfo, error) { + ret := _m.Called(ctx, bucket, prefix, count) + + if len(ret) == 0 { + panic("no return value specified for ListObjects") + } + + var r0 []fs.FileInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, int) ([]fs.FileInfo, error)); ok { + return rf(ctx, bucket, prefix, count) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, int) []fs.FileInfo); ok { + r0 = rf(ctx, bucket, prefix, count) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]fs.FileInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, int) error); ok { + r1 = rf(ctx, bucket, prefix, count) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// PutObject provides a mock function with given fields: ctx, bucket, name, reader +func (_m *ObjectManager) PutObject(ctx context.Context, bucket string, name string, reader io.Reader) (bool, error) { + ret := _m.Called(ctx, bucket, name, reader) + + if len(ret) == 0 { + panic("no return value specified for PutObject") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, io.Reader) (bool, error)); ok { + return rf(ctx, bucket, name, reader) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, io.Reader) bool); ok { + r0 = rf(ctx, bucket, name, reader) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, io.Reader) error); ok { + r1 = rf(ctx, bucket, name, reader) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewObjectManager creates a new instance of ObjectManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewObjectManager(t interface { + mock.TestingT + Cleanup(func()) +}) *ObjectManager { + mock := &ObjectManager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/ossfs/internal/utils/contract.go b/ossfs/internal/utils/contract.go new file mode 100644 index 00000000..9c458ec0 --- /dev/null +++ b/ossfs/internal/utils/contract.go @@ -0,0 +1,21 @@ +package utils + +import ( + "context" + "io" + "os" +) + +type CleanUp func() + +type ObjectManager interface { + GetObject(ctx context.Context, bucket, name string) (io.Reader, CleanUp, error) + GetObjectPart(ctx context.Context, bucket, name string, start, end int64) (io.Reader, CleanUp, error) + DeleteObject(ctx context.Context, bucket, name string) error + IsObjectExist(ctx context.Context, bucket, name string) (bool, error) + PutObject(ctx context.Context, bucket, name string, reader io.Reader) (bool, error) + CopyObject(ctx context.Context, bucket, srcName, targetName string) error + GetObjectMeta(ctx context.Context, bucket, name string) (os.FileInfo, error) + ListObjects(ctx context.Context, bucket, prefix string, count int) ([]os.FileInfo, error) + ListAllObjects(ctx context.Context, bucket, prefix string) ([]os.FileInfo, error) +} diff --git a/ossfs/internal/utils/init.go b/ossfs/internal/utils/init.go new file mode 100644 index 00000000..72709b88 --- /dev/null +++ b/ossfs/internal/utils/init.go @@ -0,0 +1,6 @@ +package utils + +func init() { + // Ensure OssObjectManager implements ObjectManager interface + var _ ObjectManager = (*OssObjectManager)(nil) +} diff --git a/ossfs/internal/utils/oss.go b/ossfs/internal/utils/oss.go new file mode 100644 index 00000000..6014d71d --- /dev/null +++ b/ossfs/internal/utils/oss.go @@ -0,0 +1,210 @@ +package utils + +import ( + "context" + "fmt" + "io" + "io/fs" + "os" + "strings" + "time" + + "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" + "github.com/spf13/afero" +) + +var ossDirSeparator string = "/" +var ossDefaultFileMode fs.FileMode = 0o755 + +type OssObjectManager struct { + ObjectManager + Client *oss.Client +} + +func (m *OssObjectManager) GetObject(ctx context.Context, bucket, name string) (io.Reader, CleanUp, error) { + req := &oss.GetObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(name), + } + res, err := m.Client.GetObject(ctx, req) + cleanUp := func() { + res.Body.Close() + } + return res.Body, cleanUp, err +} + +func (m *OssObjectManager) GetObjectPart(ctx context.Context, bucket, name string, start, end int64) (io.Reader, CleanUp, error) { + if start > end { + return nil, nil, afero.ErrOutOfRange + } + req := &oss.GetObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(name), + Range: oss.Ptr(fmt.Sprintf("bytes=%v-%v", start, end)), + RangeBehavior: oss.Ptr("standard"), + } + res, err := m.Client.GetObject(ctx, req) + cleanUp := func() { + res.Body.Close() + } + return res.Body, cleanUp, err +} + +func (m *OssObjectManager) DeleteObject(ctx context.Context, bucket, name string) error { + req := &oss.DeleteObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(name), + } + _, err := m.Client.DeleteObject(ctx, req) + return err +} + +func (m *OssObjectManager) IsObjectExist(ctx context.Context, bucket, name string) (bool, error) { + return m.Client.IsObjectExist(ctx, bucket, name) +} + +func (m *OssObjectManager) PutObject(ctx context.Context, bucket, name string, reader io.Reader) (bool, error) { + req := &oss.PutObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(name), + Body: reader, + } + _, err := m.Client.PutObject(ctx, req) + if err != nil { + return false, err + } + return true, nil +} + +func (m *OssObjectManager) CopyObject(ctx context.Context, bucket, srcName, targetName string) error { + req := &oss.CopyObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(srcName), + SourceKey: oss.Ptr(targetName), + SourceBucket: oss.Ptr(bucket), + StorageClass: oss.StorageClassStandard, + } + _, err := m.Client.CopyObject(ctx, req) + return err +} + +func (m *OssObjectManager) GetObjectMeta(ctx context.Context, bucket, name string) (os.FileInfo, error) { + req := &oss.HeadObjectRequest{ + Bucket: oss.Ptr(bucket), + Key: oss.Ptr(name), + } + + res, err := m.Client.HeadObject(ctx, req) + if err != nil { + return nil, err + } + return &OssObjectMeta{ + name: name, + size: res.ContentLength, + lastModifiedAt: *res.LastModified, + }, nil +} + +func (m *OssObjectManager) ListObjects(ctx context.Context, bucket, prefix string, count int) ([]os.FileInfo, error) { + req := &oss.ListObjectsV2Request{ + Bucket: oss.Ptr(bucket), + Delimiter: oss.Ptr(ossDirSeparator), + Prefix: oss.Ptr(prefix), + } + p := m.Client.NewListObjectsV2Paginator(req) + + s := make([]os.FileInfo, 0) + + var i int + +loop: + for p.HasNext() { + page, err := p.NextPage(ctx) + if err != nil { + return nil, err + } + + for _, obj := range page.Contents { + i++ + if i == count { + break loop + } + s = append(s, &OssObjectMeta{ + name: oss.ToString(obj.Key), + size: obj.Size, + lastModifiedAt: oss.ToTime(obj.LastModified), + }) + } + } + + return s, nil +} + +func (m *OssObjectManager) ListAllObjects(ctx context.Context, bucket, prefix string) ([]os.FileInfo, error) { + req := &oss.ListObjectsV2Request{ + Bucket: oss.Ptr(bucket), + Prefix: oss.Ptr(prefix), + } + p := m.Client.NewListObjectsV2Paginator(req) + + s := make([]os.FileInfo, 0) + + for p.HasNext() { + page, err := p.NextPage(ctx) + if err != nil { + return nil, err + } + + for _, obj := range page.Contents { + s = append(s, &OssObjectMeta{ + name: oss.ToString(obj.Key), + size: obj.Size, + lastModifiedAt: oss.ToTime(obj.LastModified), + }) + } + } + + return s, nil +} + +type OssObjectMeta struct { + os.FileInfo + name string + size int64 + lastModifiedAt time.Time +} + +func NewOssObjectMeta(name string, size int64, updatedAt time.Time) *OssObjectMeta { + return &OssObjectMeta{ + name: name, + size: size, + lastModifiedAt: updatedAt, + } +} + +func (objMeta *OssObjectMeta) isDir() bool { + return strings.HasSuffix(objMeta.name, ossDirSeparator) +} + +func (objMeta *OssObjectMeta) ModTime() time.Time { + return objMeta.lastModifiedAt +} + +func (objMeta *OssObjectMeta) Mode() fs.FileMode { + if objMeta.isDir() { + return ossDefaultFileMode | fs.ModeDir + } + return ossDefaultFileMode +} + +func (objMeta *OssObjectMeta) Name() string { + return objMeta.name +} + +func (objMeta *OssObjectMeta) Size() int64 { + return objMeta.size +} + +func (objMeta *OssObjectMeta) Sys() any { + return nil +} From 100a14e0c515e513c7b4faf4a97a6787c3199c86 Mon Sep 17 00:00:00 2001 From: messikiller Date: Mon, 14 Apr 2025 16:26:58 +0800 Subject: [PATCH 2/2] replace ossfs as third party link --- README.md | 4 + go.mod | 14 +- go.sum | 16 - ossfs/file.go | 288 ---------------- ossfs/file_info.go | 17 - ossfs/file_test.go | 296 ----------------- ossfs/fs.go | 188 ----------- ossfs/fs_test.go | 462 -------------------------- ossfs/fs_utils.go | 36 -- ossfs/init.go | 14 - ossfs/internal/mocks/FileInfo.go | 139 -------- ossfs/internal/mocks/ObjectManager.go | 293 ---------------- ossfs/internal/utils/contract.go | 21 -- ossfs/internal/utils/init.go | 6 - ossfs/internal/utils/oss.go | 210 ------------ 15 files changed, 5 insertions(+), 1999 deletions(-) delete mode 100644 ossfs/file.go delete mode 100644 ossfs/file_info.go delete mode 100644 ossfs/file_test.go delete mode 100644 ossfs/fs.go delete mode 100644 ossfs/fs_test.go delete mode 100644 ossfs/fs_utils.go delete mode 100644 ossfs/init.go delete mode 100644 ossfs/internal/mocks/FileInfo.go delete mode 100644 ossfs/internal/mocks/ObjectManager.go delete mode 100644 ossfs/internal/utils/contract.go delete mode 100644 ossfs/internal/utils/init.go delete mode 100644 ossfs/internal/utils/oss.go diff --git a/README.md b/README.md index 86f15455..b67c2059 100644 --- a/README.md +++ b/README.md @@ -399,6 +399,10 @@ implement: * SSH * S3 +## Third-party library + +- Alibaba Cloud OSS: [messikiller/afero-oss](https://github.com/messikiller/afero-oss) + # About the project ## What's in the name diff --git a/go.mod b/go.mod index fa24da40..101c2865 100644 --- a/go.mod +++ b/go.mod @@ -2,16 +2,4 @@ module github.com/spf13/afero go 1.23.0 -require ( - github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1 - github.com/stretchr/testify v1.10.0 - golang.org/x/text v0.23.0 -) - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/objx v0.5.2 // indirect - golang.org/x/time v0.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) +require golang.org/x/text v0.23.0 diff --git a/go.sum b/go.sum index 1862dd3d..d00bb390 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,2 @@ -github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1 h1:sOhpJdR/+lbQniznp3cYSfwQlXbVkT0ccuiZScBrI6Y= -github.com/aliyun/alibabacloud-oss-go-sdk-v2 v1.2.1/go.mod h1:FTzydeQVmR24FI0D6XWUOMKckjXehM/jgMn1xC+DA9M= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= -golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ossfs/file.go b/ossfs/file.go deleted file mode 100644 index 630beff8..00000000 --- a/ossfs/file.go +++ /dev/null @@ -1,288 +0,0 @@ -package ossfs - -import ( - "errors" - "io" - "os" - "strconv" - "syscall" - - "github.com/spf13/afero" -) - -type File struct { - name string - fs *Fs - openFlag int - offset int64 - fi os.FileInfo - dirty bool - closed bool - isDir bool - preloaded bool - preloadedFd afero.File -} - -func NewOssFile(name string, flag int, fs *Fs) (*File, error) { - return &File{ - name: fs.normFileName(name), - fs: fs, - openFlag: flag, - offset: 0, - dirty: false, - closed: false, - isDir: fs.isDir(fs.normFileName(name)), - preloaded: false, - preloadedFd: nil, - }, nil -} - -func (f *File) preload() error { - pfs := f.fs.preloadFs - if _, err := pfs.Stat(f.name); err == nil { - if e := pfs.Remove(f.name); e != nil { - return e - } - } - pfd, err := f.fs.preloadFs.Create(f.name) - if err != nil { - return err - } - - r, clean, e := f.fs.manager.GetObject(f.fs.ctx, f.fs.bucketName, f.name) - if e != nil { - return e - } - defer clean() - - if _, err := io.Copy(pfd, r); err != nil { - return err - } - - if _, err := pfd.Seek(f.offset, io.SeekStart); err != nil { - return err - } - - f.preloadedFd = pfd - f.preloaded = true - return nil -} - -func (f *File) freshFileInfo() error { - fi, err := f.fs.Stat(f.name) - if err != nil { - return err - } - f.fi = fi - f.dirty = false - return nil -} - -func (f *File) getFileInfo() (os.FileInfo, error) { - if f.dirty { - if err := f.freshFileInfo(); err != nil { - return nil, err - } - } - return f.fi, nil -} - -func (f *File) isReadable() bool { - return !f.closed && (f.openFlag == os.O_RDONLY || f.openFlag == os.O_RDWR) -} - -func (f *File) isWriteable() bool { - return !f.closed && (f.openFlag == os.O_WRONLY || f.openFlag == os.O_RDWR) -} - -func (f *File) isAppendOnly() bool { - return f.isWriteable() && f.openFlag&os.O_APPEND != 0 -} - -func (f *File) Read(p []byte) (int, error) { - if !f.isReadable() || f.isDir { - return 0, syscall.EPERM - } - n, err := f.ReadAt(p, f.offset) - if err != nil { - return 0, err - } - f.offset += int64(n) - return n, err -} - -func (f *File) ReadAt(p []byte, off int64) (int, error) { - if !f.isReadable() || f.isDir { - return 0, syscall.EPERM - } - reader, cleanUp, err := f.fs.manager.GetObjectPart(f.fs.ctx, f.fs.bucketName, f.name, off, off+int64(len(p))) - if err != nil { - return 0, err - } - defer cleanUp() - return reader.Read(p) -} - -func (f *File) Seek(offset int64, whence int) (int64, error) { - if (!f.isReadable() && !f.isWriteable()) || f.isDir { - return 0, syscall.EPERM - } - fi, err := f.getFileInfo() - if err != nil { - return 0, err - } - max := fi.Size() - var newOffset int64 - switch whence { - case io.SeekCurrent: - newOffset = f.offset + offset - case io.SeekStart: - newOffset = offset - case io.SeekEnd: - newOffset = max + offset - default: - return 0, errors.New("Invalid whence value: " + strconv.Itoa(whence)) - } - if newOffset < 0 || newOffset > max { - return 0, afero.ErrOutOfRange - } - f.offset = newOffset - return f.offset, nil -} - -func (f *File) doAppend(p []byte) (int, error) { - if !f.isWriteable() { - return 0, syscall.EPERM - } - fi, err := f.getFileInfo() - if err != nil { - return 0, err - } - return f.doWriteAt(p, fi.Size()) -} - -func (f *File) Write(p []byte) (int, error) { - if !f.isWriteable() { - return 0, syscall.EPERM - } - if f.isAppendOnly() { - return f.doAppend(p) - } - n, e := f.doWriteAt(p, f.offset) - if e != nil { - return 0, e - } - f.offset += int64(n) - return n, e -} - -func (f *File) doWriteAt(p []byte, off int64) (int, error) { - if f.isDir { - return 0, syscall.EPERM - } - - if !f.preloaded { - if err := f.preload(); err != nil { - return 0, err - } - } - - n, e := f.preloadedFd.WriteAt(p, off) - f.dirty = true - if f.fs.autoSync { - f.Sync() - } - return n, e -} - -func (f *File) WriteAt(p []byte, off int64) (int, error) { - if !f.isWriteable() || f.isAppendOnly() { - return 0, syscall.EPERM - } - return f.doWriteAt(p, off) -} - -func (f *File) Close() error { - f.Sync() - f.closed = true - delete(f.fs.openedFiles, f.name) - if f.preloaded { - err := f.fs.preloadFs.Remove(f.name) - if err != nil { - return err - } - err = f.preloadedFd.Close() - if err != nil { - return err - } - f.preloadedFd = nil - f.preloaded = false - } - return nil -} - -func (f *File) Name() string { - return f.name -} - -func (f *File) Readdir(count int) ([]os.FileInfo, error) { - if !f.isReadable() { - return nil, syscall.EPERM - } - - fis, err := f.fs.manager.ListObjects(f.fs.ctx, f.fs.bucketName, f.fs.ensureAsDir(f.name), count) - return fis, err -} - -func (f *File) Readdirnames(n int) ([]string, error) { - if !f.isReadable() { - return nil, syscall.EPERM - } - - fis, err := f.Readdir(n) - if err != nil { - return nil, err - } - var fNames []string - for _, fi := range fis { - fNames = append(fNames, fi.Name()) - } - - return fNames, nil -} - -func (f *File) Stat() (os.FileInfo, error) { - if f.dirty { - err := f.freshFileInfo() - if err != nil { - return nil, err - } - } - return f.fi, nil -} - -func (f *File) Sync() error { - if f.preloaded { - if _, err := f.fs.manager.PutObject(f.fs.ctx, f.fs.bucketName, f.name, f.preloadedFd); err != nil { - return err - } - } - if f.dirty { - if err := f.freshFileInfo(); err != nil { - return err - } - } - return nil -} - -func (f *File) Truncate(size int64) error { - if !f.isWriteable() || f.isDir { - return syscall.EPERM - } - _, err := f.WriteAt([]byte(""), 0) - return err -} - -func (f *File) WriteString(s string) (int, error) { - return f.Write([]byte(s)) -} diff --git a/ossfs/file_info.go b/ossfs/file_info.go deleted file mode 100644 index ea31d2da..00000000 --- a/ossfs/file_info.go +++ /dev/null @@ -1,17 +0,0 @@ -package ossfs - -import ( - "time" - - "github.com/spf13/afero/ossfs/internal/utils" -) - -type FileInfo struct { - *utils.OssObjectMeta -} - -func NewFileInfo(name string, size int64, updatedAt time.Time) *FileInfo { - return &FileInfo{ - OssObjectMeta: utils.NewOssObjectMeta(name, size, updatedAt), - } -} diff --git a/ossfs/file_test.go b/ossfs/file_test.go deleted file mode 100644 index fba21bd2..00000000 --- a/ossfs/file_test.go +++ /dev/null @@ -1,296 +0,0 @@ -package ossfs - -import ( - "io" - "os" - "strings" - "syscall" - "testing" - "time" - - "github.com/spf13/afero" - "github.com/spf13/afero/ossfs/internal/mocks" - "github.com/spf13/afero/ossfs/internal/utils" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func getMockedFs() *Fs { - fs := NewOssFs("test-ak", "test-sk", "test-region", "test-bucket") - fs.manager = &mocks.ObjectManager{} - return fs -} - -func getMockedFile(name string, flag int, fs *Fs) *File { - f, _ := NewOssFile(name, flag, fs) - return f -} - -func TestNewOssFile(t *testing.T) { - t.Run("create new file with read flag", func(t *testing.T) { - fs := &Fs{} - file, err := NewOssFile("testfile", os.O_RDONLY, fs) - assert.NoError(t, err) - assert.Equal(t, "testfile", file.name) - assert.Equal(t, os.O_RDONLY, file.openFlag) - assert.Equal(t, fs, file.fs) - assert.False(t, file.dirty) - assert.False(t, file.closed) - assert.False(t, file.isDir) - assert.False(t, file.preloaded) - assert.Nil(t, file.preloadedFd) - }) - - t.Run("create new file with write flag", func(t *testing.T) { - fs := &Fs{} - file, err := NewOssFile("testfile", os.O_WRONLY, fs) - assert.NoError(t, err) - assert.Equal(t, "testfile", file.name) - assert.Equal(t, os.O_WRONLY, file.openFlag) - assert.Equal(t, fs, file.fs) - assert.False(t, file.dirty) - assert.False(t, file.closed) - assert.False(t, file.isDir) - assert.False(t, file.preloaded) - assert.Nil(t, file.preloadedFd) - }) - - t.Run("create new directory", func(t *testing.T) { - fs := &Fs{} - file, err := NewOssFile("testdir/", os.O_RDONLY, fs) - assert.NoError(t, err) - assert.Equal(t, "testdir/", file.name) - assert.Equal(t, os.O_RDONLY, file.openFlag) - assert.Equal(t, fs, file.fs) - assert.False(t, file.dirty) - assert.False(t, file.closed) - assert.True(t, file.isDir) - assert.False(t, file.preloaded) - assert.Nil(t, file.preloadedFd) - }) - - t.Run("normalize file name", func(t *testing.T) { - fs := &Fs{} - file, err := NewOssFile("/path/testfile", os.O_RDONLY, fs) - assert.NoError(t, err) - assert.Equal(t, "path/testfile", file.name) - }) -} - -func TestRead(t *testing.T) { - t.Run("Read with unreadable flag return error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_WRONLY, fs) - - p := make([]byte, 0) - _, e := f.Read(p) - - assert.Error(t, e) - assert.NotNil(t, e) - }) - - t.Run("Read on directory return error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testdir", os.O_RDONLY, fs) - f.isDir = true - - p := make([]byte, 10) - _, e := f.Read(p) - - assert.Error(t, e) - assert.Equal(t, syscall.EPERM, e) - }) - - t.Run("Read on closed file return error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDONLY, fs) - f.closed = true - - p := make([]byte, 10) - _, e := f.Read(p) - - assert.Error(t, e) - assert.Equal(t, syscall.EPERM, e) - }) - - t.Run("Successful read updates offset", func(t *testing.T) { - fs := getMockedFs() - var cu utils.CleanUp = func() {} - mockManager := fs.manager.(*mocks.ObjectManager) - mockManager.On("GetObjectPart", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - // Return(&mockReadCloser{data: []byte("testdata")}, cu, nil) - Return(strings.NewReader("testdata"), cu, nil) - - f := getMockedFile("testfile", os.O_RDONLY, fs) - p := make([]byte, 8) - n, err := f.Read(p) - - assert.NoError(t, err) - assert.Equal(t, 8, n) - assert.Equal(t, int64(8), f.offset) - }) - - t.Run("ReadAt error propagates", func(t *testing.T) { - fs := getMockedFs() - mockManager := fs.manager.(*mocks.ObjectManager) - mockManager.On("GetObjectPart", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(nil, nil, syscall.EIO) - - f := getMockedFile("testfile", os.O_RDONLY, fs) - p := make([]byte, 8) - _, err := f.Read(p) - - assert.Error(t, err) - assert.Equal(t, syscall.EIO, err) - }) -} - -func TestReadAt(t *testing.T) { - t.Run("ReadAt with unreadable flag return error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_WRONLY, fs) - - p := make([]byte, 0) - _, e := f.ReadAt(p, 0) - - assert.Error(t, e) - assert.NotNil(t, e) - }) - - t.Run("ReadAt on dir return error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("/path/to/dir/", os.O_WRONLY, fs) - - p := make([]byte, 0) - _, e := f.ReadAt(p, 0) - - assert.Error(t, e) - assert.NotNil(t, e) - }) - - t.Run("ReadAt success", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDONLY, fs) - - p := make([]byte, 4) - - var cu utils.CleanUp = func() {} - off := int64(5) - m := &mocks.ObjectManager{} - m. - On("GetObjectPart", f.fs.ctx, f.fs.bucketName, f.name, off, off+int64(len(p))). - Return(strings.NewReader("test result"), cu, nil) - fs.manager = m - - n, e := f.ReadAt(p, off) - - assert.Nil(t, e) - assert.Equal(t, 4, n) - assert.Equal(t, "test", string(p)) - }) -} - -func TestSeek(t *testing.T) { - t.Run("Seek on unreadable/unwritable file returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_WRONLY, fs) - f.closed = true - - _, err := f.Seek(0, io.SeekStart) - assert.Error(t, err) - assert.Equal(t, syscall.EPERM, err) - }) - - t.Run("Seek on directory returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testdir", os.O_RDONLY, fs) - f.isDir = true - - _, err := f.Seek(0, io.SeekStart) - assert.Error(t, err) - assert.Equal(t, syscall.EPERM, err) - }) - - t.Run("SeekStart sets correct offset", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - - offset, err := f.Seek(10, io.SeekStart) - assert.NoError(t, err) - assert.Equal(t, int64(10), offset) - assert.Equal(t, int64(10), f.offset) - }) - - t.Run("SeekCurrent adjusts offset correctly", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - f.offset = 5 - - offset, err := f.Seek(5, io.SeekCurrent) - assert.NoError(t, err) - assert.Equal(t, int64(10), offset) - assert.Equal(t, int64(10), f.offset) - }) - - t.Run("SeekEnd adjusts offset correctly", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - - offset, err := f.Seek(-10, io.SeekEnd) - assert.NoError(t, err) - assert.Equal(t, int64(90), offset) - assert.Equal(t, int64(90), f.offset) - }) - - t.Run("Seek beyond file size returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - - _, err := f.Seek(101, io.SeekStart) - assert.Error(t, err) - assert.Equal(t, afero.ErrOutOfRange, err) - }) - - t.Run("Seek negative offset returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - - _, err := f.Seek(-1, io.SeekStart) - assert.Error(t, err) - assert.Equal(t, afero.ErrOutOfRange, err) - }) - - t.Run("Seek with invalid whence returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDWR, fs) - f.fi = NewFileInfo("testfile", 100, time.Now()) - - _, err := f.Seek(0, 3) - assert.Error(t, err) - }) -} - -func TestWrite(t *testing.T) { - t.Run("Write unwritable file returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("testfile", os.O_RDONLY, fs) - _, e := f.Write([]byte("test input string")) - - assert.Error(t, e) - assert.NotNil(t, e) - }) - - t.Run("Write dir returns error", func(t *testing.T) { - fs := getMockedFs() - f := getMockedFile("/path/to/test_dir/", os.O_WRONLY, fs) - _, e := f.Write([]byte("test input string")) - - assert.Error(t, e) - assert.NotNil(t, e) - }) -} diff --git a/ossfs/fs.go b/ossfs/fs.go deleted file mode 100644 index 05ef2ffd..00000000 --- a/ossfs/fs.go +++ /dev/null @@ -1,188 +0,0 @@ -package ossfs - -import ( - "context" - "errors" - "os" - "strings" - "time" - - "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" - "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials" - "github.com/spf13/afero" - "github.com/spf13/afero/ossfs/internal/utils" -) - -const ( - defaultFileMode = 0o755 - defaultFileFlag = os.O_RDWR -) - -type Fs struct { - manager utils.ObjectManager - bucketName string - separator string - autoSync bool - openedFiles map[string]afero.File - preloadFs afero.Fs - ctx context.Context -} - -func NewOssFs(accessKeyId, accessKeySecret, region, bucket string) *Fs { - ossCfg := oss.LoadDefaultConfig(). - WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyId, accessKeySecret)). - WithRegion(region) - - return &Fs{ - manager: &utils.OssObjectManager{ - Client: oss.NewClient(ossCfg), - }, - bucketName: bucket, - separator: "/", - autoSync: true, - openedFiles: make(map[string]afero.File), - preloadFs: afero.NewMemMapFs(), - ctx: context.Background(), - } -} - -func (fs *Fs) WithContext(ctx context.Context) *Fs { - fs.ctx = ctx - return fs -} - -// Create creates a new empty file and open it, return the open file and error -// if any happens. -func (fs *Fs) Create(name string) (afero.File, error) { - n := fs.normFileName(name) - r := strings.NewReader("") - if _, err := fs.manager.PutObject(fs.ctx, fs.bucketName, n, r); err != nil { - return nil, err - } - return NewOssFile(n, defaultFileFlag, fs) -} - -// Mkdir creates a directory in the filesystem, return an error if any -// happens. -func (fs *Fs) Mkdir(name string, perm os.FileMode) error { - return fs.MkdirAll(fs.ensureAsDir(name), perm) -} - -// MkdirAll creates a directory path and all parents that does not exist -// yet. -func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { - dirName := fs.ensureAsDir(path) - r := strings.NewReader("") - _, err := fs.manager.PutObject(fs.ctx, fs.bucketName, dirName, r) - return err -} - -// Open opens a file, returning it or an error, if any happens. -func (fs *Fs) Open(name string) (afero.File, error) { - return fs.OpenFile(name, defaultFileFlag, defaultFileMode) -} - -// OpenFile opens a file using the given flags and the given mode. -func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { - name = fs.normFileName(name) - file, found := fs.openedFiles[name] - if found && file.(*File).openFlag == flag { - return file, nil - } - - f, err := NewOssFile(name, flag, fs) - if err != nil { - return nil, err - } - - existed := false - existed, err = fs.manager.IsObjectExist(fs.ctx, fs.bucketName, name) - if err != nil { - return nil, err - } - - if !existed && f.openFlag&os.O_CREATE == 0 { - return nil, afero.ErrFileNotFound - } - - if !existed && f.openFlag*os.O_CREATE != 0 { - if _, err := fs.Create(f.name); err != nil { - return nil, err - } - } - - if f.openFlag&os.O_TRUNC != 0 { - _, err := f.fs.manager.PutObject(fs.ctx, fs.bucketName, f.name, strings.NewReader("")) - if err != nil { - return nil, err - } - } - - fs.openedFiles[name] = f - - return f, nil -} - -// Remove removes a file identified by name, returning an error, if any -// happens. -func (fs *Fs) Remove(name string) error { - return fs.manager.DeleteObject(fs.ctx, fs.bucketName, fs.normFileName(name)) -} - -// RemoveAll removes a directory path and any children it contains. It -// does not fail if the path does not exist (return nil). -func (fs *Fs) RemoveAll(path string) error { - dir := fs.ensureAsDir(path) - fis, err := fs.manager.ListAllObjects(fs.ctx, fs.bucketName, dir) - if err != nil { - return err - } - for _, fi := range fis { - err = fs.manager.DeleteObject(fs.ctx, fs.bucketName, fi.Name()) - if err != nil { - return err - } - } - return nil -} - -// Rename renames a file. -func (fs *Fs) Rename(oldname, newname string) error { - err := fs.manager.CopyObject(fs.ctx, fs.bucketName, oldname, newname) - if err != nil { - return err - } - err = fs.manager.DeleteObject(fs.ctx, fs.bucketName, oldname) - return err -} - -// Stat returns a FileInfo describing the named file, or an error, if any -// happens. -func (fs *Fs) Stat(name string) (os.FileInfo, error) { - fi, err := fs.manager.GetObjectMeta(fs.ctx, fs.bucketName, fs.normFileName(name)) - if err != nil { - return nil, err - } - - return fi, err -} - -// The name of this FileSystem -func (fs *Fs) Name() string { - return "OssFs" -} - -// Chmod changes the mode of the named file to mode. -func (fs *Fs) Chmod(name string, mode os.FileMode) error { - return errors.New("OSS: method Chmod is not implemented") -} - -// Chown changes the uid and gid of the named file. -func (fs *Fs) Chown(name string, uid, gid int) error { - return errors.New("OSS: method Chown is not implemented") -} - -// Chtimes changes the access and modification times of the named file -func (fs *Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { - return errors.New("OSS: method Chtimes is not implemented") -} diff --git a/ossfs/fs_test.go b/ossfs/fs_test.go deleted file mode 100644 index 8d09ecf8..00000000 --- a/ossfs/fs_test.go +++ /dev/null @@ -1,462 +0,0 @@ -package ossfs - -import ( - "context" - "os" - "strings" - "testing" - "time" - - "github.com/spf13/afero" - "github.com/spf13/afero/ossfs/internal/mocks" - "github.com/stretchr/testify/assert" -) - -func TestNewOssFs(t *testing.T) { - tests := []struct { - name string - accessKeyId string - accessKeySecret string - region string - bucket string - expected *Fs - }{ - { - name: "valid credentials", - accessKeyId: "testKeyId", - accessKeySecret: "testKeySecret", - region: "test-region", - bucket: "test-bucket", - expected: &Fs{ - bucketName: "test-bucket", - separator: "/", - autoSync: true, - openedFiles: make(map[string]afero.File), - preloadFs: afero.NewMemMapFs(), - ctx: context.Background(), - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := NewOssFs(tt.accessKeyId, tt.accessKeySecret, tt.region, tt.bucket) - assert.NotNil(t, got.manager) - assert.Equal(t, tt.expected.bucketName, got.bucketName) - assert.Equal(t, tt.expected.separator, got.separator) - assert.Equal(t, tt.expected.autoSync, got.autoSync) - assert.NotNil(t, got.openedFiles) - assert.NotNil(t, got.preloadFs) - assert.NotNil(t, got.ctx) - }) - } -} - -func TestFsWithContext(t *testing.T) { - type bgMeta string - tests := []struct { - name string - fs *Fs - ctx context.Context - expected *Fs - }{ - { - name: "set new context", - fs: &Fs{ - ctx: context.Background(), - }, - ctx: context.WithValue(context.Background(), bgMeta("testKey"), bgMeta("testValue")), - expected: &Fs{ - ctx: context.WithValue(context.Background(), bgMeta("testKey"), bgMeta("testValue")), - }, - }, - { - name: "set nil context", - fs: &Fs{ - ctx: context.Background(), - }, - ctx: nil, - expected: &Fs{ - ctx: nil, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := tt.fs.WithContext(tt.ctx) - assert.Equal(t, tt.expected.ctx, got.ctx) - assert.Equal(t, tt.fs, got) - }) - } -} - -func TestFsCreate(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("create simple success", func(t *testing.T) { - m.On("PutObject", fs.ctx, bucket, "test.txt", strings.NewReader("")).Return(true, nil).Once() - file, err := fs.Create("test.txt") - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - m.AssertExpectations(t) - }) - - t.Run("create prefixed file path success", func(t *testing.T) { - m. - On("PutObject", fs.ctx, bucket, "path/to/test.txt", strings.NewReader("")). - Return(true, nil). - Once() - file, err := fs.Create("/path/to/test.txt") - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - m.AssertExpectations(t) - }) - - t.Run("create dir path success", func(t *testing.T) { - m. - On("PutObject", ctx, bucket, "path/to/test_dir/", strings.NewReader("")). - Return(true, nil). - Once() - file, err := fs.Create("/path/to/test_dir/") - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - assert.Equal(t, "path/to/test_dir/", file.Name()) - m.AssertExpectations(t) - }) - - t.Run("create failure", func(t *testing.T) { - m.On("PutObject", fs.ctx, bucket, "test2.txt", strings.NewReader("")).Return(false, afero.ErrFileNotFound).Once() - _, err := fs.Create("test2.txt") - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) -} - -func TestFsMkdirAll(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("MkDirAll simple success", func(t *testing.T) { - m. - On("PutObject", fs.ctx, bucket, "path/to/test_dir/", strings.NewReader("")). - Return(true, nil). - Once() - err := fs.MkdirAll("/path/to/test_dir/", defaultFileMode) - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("MkDirAll failure", func(t *testing.T) { - m. - On("PutObject", fs.ctx, bucket, "path/to/test_dir/", strings.NewReader("")). - Return(false, afero.ErrFileClosed). - Once() - err := fs.MkdirAll("/path/to/test_dir/", defaultFileMode) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileClosed) - m.AssertExpectations(t) - }) -} - -func TestFsOpenFile(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - openedFiles: make(map[string]afero.File), - } - - t.Run("open existing file success", func(t *testing.T) { - m.On("IsObjectExist", ctx, bucket, "test.txt").Return(true, nil).Once() - file, err := fs.OpenFile("test.txt", os.O_RDONLY, 0644) - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - m.AssertExpectations(t) - }) - - t.Run("open non-existing file with create flag success", func(t *testing.T) { - m.On("IsObjectExist", ctx, bucket, "new.txt").Return(false, nil).Once() - m.On("PutObject", ctx, bucket, "new.txt", strings.NewReader("")).Return(true, nil).Once() - file, err := fs.OpenFile("new.txt", os.O_CREATE|os.O_RDWR, 0644) - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - m.AssertExpectations(t) - }) - - t.Run("open file with truncate flag success", func(t *testing.T) { - m.On("IsObjectExist", ctx, bucket, "trunc.txt").Return(true, nil).Once() - m.On("PutObject", ctx, bucket, "trunc.txt", strings.NewReader("")).Return(true, nil).Once() - file, err := fs.OpenFile("trunc.txt", os.O_TRUNC|os.O_RDWR, 0644) - assert.Nil(t, err) - assert.NotNil(t, file) - assert.Implements(t, (*afero.File)(nil), file) - m.AssertExpectations(t) - }) - - t.Run("open existing file from cache", func(t *testing.T) { - cachedFile := &File{name: "cached.txt", openFlag: os.O_RDONLY} - fs.openedFiles["cached.txt"] = cachedFile - file, err := fs.OpenFile("cached.txt", os.O_RDONLY, 0644) - assert.Nil(t, err) - assert.Equal(t, cachedFile, file) - assert.Implements(t, (*afero.File)(nil), file) - }) - - t.Run("open non-existing file without create flag fails", func(t *testing.T) { - m.On("IsObjectExist", ctx, bucket, "nonexist.txt").Return(false, nil).Once() - _, err := fs.OpenFile("nonexist.txt", os.O_RDONLY, 0644) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) - - t.Run("open file with check exist error fails", func(t *testing.T) { - m.On("IsObjectExist", ctx, bucket, "error.txt").Return(false, afero.ErrFileNotFound).Once() - _, err := fs.OpenFile("error.txt", os.O_RDONLY, 0644) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) -} - -func TestFsRemove(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("remove file success", func(t *testing.T) { - m.On("DeleteObject", fs.ctx, bucket, "test.txt").Return(nil).Once() - err := fs.Remove("test.txt") - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("remove prefixed file success", func(t *testing.T) { - m.On("DeleteObject", fs.ctx, bucket, "path/to/test.txt").Return(nil).Once() - err := fs.Remove("/path/to/test.txt") - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("remove non-existent file", func(t *testing.T) { - m.On("DeleteObject", fs.ctx, bucket, "nonexistent.txt").Return(afero.ErrFileNotFound).Once() - err := fs.Remove("nonexistent.txt") - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) -} - -func TestFsRemoveAll(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("remove non-empty directory", func(t *testing.T) { - dirPath := "path/to/dir/" - files := []os.FileInfo{ - NewFileInfo("path/to/dir/file1.txt", 100, time.Now()), - NewFileInfo("path/to/dir/file2.txt", 200, time.Now()), - NewFileInfo("path/to/dir/subdir/", 0, time.Now()), - } - - m.On("ListAllObjects", ctx, bucket, dirPath).Return(files, nil).Once() - m.On("DeleteObject", ctx, bucket, "path/to/dir/file1.txt").Return(nil).Once() - m.On("DeleteObject", ctx, bucket, "path/to/dir/file2.txt").Return(nil).Once() - m.On("DeleteObject", ctx, bucket, "path/to/dir/subdir/").Return(nil).Once() - - err := fs.RemoveAll(dirPath) - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("remove empty directory", func(t *testing.T) { - dirPath := "empty/dir/" - m.On("ListAllObjects", ctx, bucket, dirPath).Return([]os.FileInfo{}, nil).Once() - - err := fs.RemoveAll(dirPath) - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("remove non-existent path", func(t *testing.T) { - nonExistentPath := "nonexistent/path/" - m.On("ListAllObjects", ctx, bucket, nonExistentPath).Return([]os.FileInfo{}, nil).Once() - - err := fs.RemoveAll(nonExistentPath) - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("list objects failure", func(t *testing.T) { - dirPath := "path/to/dir/" - m.On("ListAllObjects", ctx, bucket, dirPath).Return(nil, afero.ErrFileNotFound).Once() - - err := fs.RemoveAll(dirPath) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) - - t.Run("delete object failure", func(t *testing.T) { - dirPath := "path/to/dir/" - files := []os.FileInfo{ - NewFileInfo("path/to/dir/file1.txt", 0, time.Now()), - } - - m.On("ListAllObjects", ctx, bucket, dirPath).Return(files, nil).Once() - m.On("DeleteObject", ctx, bucket, "path/to/dir/file1.txt").Return(afero.ErrFileNotFound).Once() - - err := fs.RemoveAll(dirPath) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) -} - -func TestFsRename(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("successful rename", func(t *testing.T) { - oldname := "old/file.txt" - newname := "new/file.txt" - - m.On("CopyObject", ctx, bucket, oldname, newname).Return(nil).Once() - m.On("DeleteObject", ctx, bucket, oldname).Return(nil).Once() - - err := fs.Rename(oldname, newname) - assert.Nil(t, err) - m.AssertExpectations(t) - }) - - t.Run("copy failure", func(t *testing.T) { - oldname := "old/file.txt" - newname := "new/file.txt" - - m.On("CopyObject", ctx, bucket, oldname, newname).Return(afero.ErrFileNotFound).Once() - - err := fs.Rename(oldname, newname) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) - - t.Run("delete failure after successful copy", func(t *testing.T) { - oldname := "old/file.txt" - newname := "new/file.txt" - - m.On("CopyObject", ctx, bucket, oldname, newname).Return(nil).Once() - m.On("DeleteObject", ctx, bucket, oldname).Return(afero.ErrFileNotFound).Once() - - err := fs.Rename(oldname, newname) - assert.NotNil(t, err) - assert.ErrorIs(t, err, afero.ErrFileNotFound) - m.AssertExpectations(t) - }) -} - -func TestFsStat(t *testing.T) { - m := &mocks.ObjectManager{} - bucket := "test-bucket" - ctx := context.TODO() - - fs := &Fs{ - manager: m, - bucketName: bucket, - ctx: ctx, - separator: "/", - } - - t.Run("stat file success", func(t *testing.T) { - expectedInfo := &mocks.FileInfo{} - m.On("GetObjectMeta", fs.ctx, bucket, "test.txt").Return(expectedInfo, nil).Once() - info, err := fs.Stat("test.txt") - assert.Nil(t, err) - assert.Equal(t, expectedInfo, info) - m.AssertExpectations(t) - }) - - t.Run("stat prefixed file path success", func(t *testing.T) { - expectedInfo := &mocks.FileInfo{} - m.On("GetObjectMeta", fs.ctx, bucket, "path/to/test.txt").Return(expectedInfo, nil).Once() - info, err := fs.Stat("/path/to/test.txt") - assert.Nil(t, err) - assert.Equal(t, expectedInfo, info) - m.AssertExpectations(t) - }) - - t.Run("stat dir path success", func(t *testing.T) { - expectedInfo := &mocks.FileInfo{} - m.On("GetObjectMeta", fs.ctx, bucket, "path/to/dir/").Return(expectedInfo, nil).Once() - info, err := fs.Stat("/path/to/dir/") - assert.Nil(t, err) - assert.Equal(t, expectedInfo, info) - m.AssertExpectations(t) - }) - - t.Run("stat non-existent file", func(t *testing.T) { - m.On("GetObjectMeta", fs.ctx, bucket, "nonexistent.txt").Return(nil, os.ErrNotExist).Once() - _, err := fs.Stat("nonexistent.txt") - assert.NotNil(t, err) - assert.ErrorIs(t, err, os.ErrNotExist) - m.AssertExpectations(t) - }) -} - -func TestFsName(t *testing.T) { - fs := &Fs{} - name := fs.Name() - assert.Equal(t, "OssFs", name) -} diff --git a/ossfs/fs_utils.go b/ossfs/fs_utils.go deleted file mode 100644 index 2a14609c..00000000 --- a/ossfs/fs_utils.go +++ /dev/null @@ -1,36 +0,0 @@ -package ossfs - -import ( - "strings" -) - -func (fs *Fs) isDir(s string) bool { - sep := fs.separator - if fs.separator == "" { - sep = "/" - } - return strings.HasSuffix(s, sep) -} - -func (fs *Fs) ensureAsDir(s string) string { - sep := fs.separator - if fs.separator == "" { - sep = "/" - } - s = fs.normFileName(s) - if !strings.HasSuffix(s, sep) { - s = s + sep - } - return s -} - -func (fs *Fs) normFileName(s string) string { - sep := fs.separator - if fs.separator == "" { - sep = "/" - } - s = strings.TrimLeft(s, "/\\") - s = strings.Replace(s, "\\", sep, -1) - s = strings.Replace(s, "/", sep, -1) - return s -} diff --git a/ossfs/init.go b/ossfs/init.go deleted file mode 100644 index 3a8ae873..00000000 --- a/ossfs/init.go +++ /dev/null @@ -1,14 +0,0 @@ -package ossfs - -import ( - "os" - - "github.com/spf13/afero" -) - -func init() { - // Ensure oss.Fs implements afero.Fs interface - var _ afero.Fs = (*Fs)(nil) - var _ afero.File = (*File)(nil) - var _ os.FileInfo = (*FileInfo)(nil) -} diff --git a/ossfs/internal/mocks/FileInfo.go b/ossfs/internal/mocks/FileInfo.go deleted file mode 100644 index 64328f2b..00000000 --- a/ossfs/internal/mocks/FileInfo.go +++ /dev/null @@ -1,139 +0,0 @@ -// Code generated by mockery v2.53.3. DO NOT EDIT. - -package mocks - -import ( - "os" - time "time" - - mock "github.com/stretchr/testify/mock" -) - -// FileInfo is an autogenerated mock type for the FileInfo type -type FileInfo struct { - mock.Mock -} - -// IsDir provides a mock function with no fields -func (_m *FileInfo) IsDir() bool { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for IsDir") - } - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// ModTime provides a mock function with no fields -func (_m *FileInfo) ModTime() time.Time { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for ModTime") - } - - var r0 time.Time - if rf, ok := ret.Get(0).(func() time.Time); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(time.Time) - } - - return r0 -} - -// Mode provides a mock function with no fields -func (_m *FileInfo) Mode() os.FileMode { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Mode") - } - - var r0 os.FileMode - if rf, ok := ret.Get(0).(func() os.FileMode); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(os.FileMode) - } - - return r0 -} - -// Name provides a mock function with no fields -func (_m *FileInfo) Name() string { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Name") - } - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -// Size provides a mock function with no fields -func (_m *FileInfo) Size() int64 { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Size") - } - - var r0 int64 - if rf, ok := ret.Get(0).(func() int64); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(int64) - } - - return r0 -} - -// Sys provides a mock function with no fields -func (_m *FileInfo) Sys() interface{} { - ret := _m.Called() - - if len(ret) == 0 { - panic("no return value specified for Sys") - } - - var r0 interface{} - if rf, ok := ret.Get(0).(func() interface{}); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(interface{}) - } - } - - return r0 -} - -// NewFileInfo creates a new instance of FileInfo. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewFileInfo(t interface { - mock.TestingT - Cleanup(func()) -}) *FileInfo { - mock := &FileInfo{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/ossfs/internal/mocks/ObjectManager.go b/ossfs/internal/mocks/ObjectManager.go deleted file mode 100644 index bb3e087b..00000000 --- a/ossfs/internal/mocks/ObjectManager.go +++ /dev/null @@ -1,293 +0,0 @@ -// Code generated by mockery v2.53.3. DO NOT EDIT. - -package mocks - -import ( - context "context" - fs "io/fs" - - io "io" - - mock "github.com/stretchr/testify/mock" - - utils "github.com/spf13/afero/ossfs/internal/utils" -) - -// ObjectManager is an autogenerated mock type for the ObjectManager type -type ObjectManager struct { - mock.Mock -} - -// CopyObject provides a mock function with given fields: ctx, bucket, srcName, targetName -func (_m *ObjectManager) CopyObject(ctx context.Context, bucket string, srcName string, targetName string) error { - ret := _m.Called(ctx, bucket, srcName, targetName) - - if len(ret) == 0 { - panic("no return value specified for CopyObject") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, string) error); ok { - r0 = rf(ctx, bucket, srcName, targetName) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// DeleteObject provides a mock function with given fields: ctx, bucket, name -func (_m *ObjectManager) DeleteObject(ctx context.Context, bucket string, name string) error { - ret := _m.Called(ctx, bucket, name) - - if len(ret) == 0 { - panic("no return value specified for DeleteObject") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { - r0 = rf(ctx, bucket, name) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetObject provides a mock function with given fields: ctx, bucket, name -func (_m *ObjectManager) GetObject(ctx context.Context, bucket string, name string) (io.Reader, utils.CleanUp, error) { - ret := _m.Called(ctx, bucket, name) - - if len(ret) == 0 { - panic("no return value specified for GetObject") - } - - var r0 io.Reader - var r1 utils.CleanUp - var r2 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) (io.Reader, utils.CleanUp, error)); ok { - return rf(ctx, bucket, name) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) io.Reader); ok { - r0 = rf(ctx, bucket, name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(io.Reader) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) utils.CleanUp); ok { - r1 = rf(ctx, bucket, name) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(utils.CleanUp) - } - } - - if rf, ok := ret.Get(2).(func(context.Context, string, string) error); ok { - r2 = rf(ctx, bucket, name) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// GetObjectMeta provides a mock function with given fields: ctx, bucket, name -func (_m *ObjectManager) GetObjectMeta(ctx context.Context, bucket string, name string) (fs.FileInfo, error) { - ret := _m.Called(ctx, bucket, name) - - if len(ret) == 0 { - panic("no return value specified for GetObjectMeta") - } - - var r0 fs.FileInfo - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) (fs.FileInfo, error)); ok { - return rf(ctx, bucket, name) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) fs.FileInfo); ok { - r0 = rf(ctx, bucket, name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(fs.FileInfo) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { - r1 = rf(ctx, bucket, name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetObjectPart provides a mock function with given fields: ctx, bucket, name, start, end -func (_m *ObjectManager) GetObjectPart(ctx context.Context, bucket string, name string, start int64, end int64) (io.Reader, utils.CleanUp, error) { - ret := _m.Called(ctx, bucket, name, start, end) - - if len(ret) == 0 { - panic("no return value specified for GetObjectPart") - } - - var r0 io.Reader - var r1 utils.CleanUp - var r2 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64) (io.Reader, utils.CleanUp, error)); ok { - return rf(ctx, bucket, name, start, end) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64) io.Reader); ok { - r0 = rf(ctx, bucket, name, start, end) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(io.Reader) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string, int64, int64) utils.CleanUp); ok { - r1 = rf(ctx, bucket, name, start, end) - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(utils.CleanUp) - } - } - - if rf, ok := ret.Get(2).(func(context.Context, string, string, int64, int64) error); ok { - r2 = rf(ctx, bucket, name, start, end) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// IsObjectExist provides a mock function with given fields: ctx, bucket, name -func (_m *ObjectManager) IsObjectExist(ctx context.Context, bucket string, name string) (bool, error) { - ret := _m.Called(ctx, bucket, name) - - if len(ret) == 0 { - panic("no return value specified for IsObjectExist") - } - - var r0 bool - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) (bool, error)); ok { - return rf(ctx, bucket, name) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) bool); ok { - r0 = rf(ctx, bucket, name) - } else { - r0 = ret.Get(0).(bool) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { - r1 = rf(ctx, bucket, name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListAllObjects provides a mock function with given fields: ctx, bucket, prefix -func (_m *ObjectManager) ListAllObjects(ctx context.Context, bucket string, prefix string) ([]fs.FileInfo, error) { - ret := _m.Called(ctx, bucket, prefix) - - if len(ret) == 0 { - panic("no return value specified for ListAllObjects") - } - - var r0 []fs.FileInfo - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]fs.FileInfo, error)); ok { - return rf(ctx, bucket, prefix) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) []fs.FileInfo); ok { - r0 = rf(ctx, bucket, prefix) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]fs.FileInfo) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { - r1 = rf(ctx, bucket, prefix) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListObjects provides a mock function with given fields: ctx, bucket, prefix, count -func (_m *ObjectManager) ListObjects(ctx context.Context, bucket string, prefix string, count int) ([]fs.FileInfo, error) { - ret := _m.Called(ctx, bucket, prefix, count) - - if len(ret) == 0 { - panic("no return value specified for ListObjects") - } - - var r0 []fs.FileInfo - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, int) ([]fs.FileInfo, error)); ok { - return rf(ctx, bucket, prefix, count) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string, int) []fs.FileInfo); ok { - r0 = rf(ctx, bucket, prefix, count) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]fs.FileInfo) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string, int) error); ok { - r1 = rf(ctx, bucket, prefix, count) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// PutObject provides a mock function with given fields: ctx, bucket, name, reader -func (_m *ObjectManager) PutObject(ctx context.Context, bucket string, name string, reader io.Reader) (bool, error) { - ret := _m.Called(ctx, bucket, name, reader) - - if len(ret) == 0 { - panic("no return value specified for PutObject") - } - - var r0 bool - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, io.Reader) (bool, error)); ok { - return rf(ctx, bucket, name, reader) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string, io.Reader) bool); ok { - r0 = rf(ctx, bucket, name, reader) - } else { - r0 = ret.Get(0).(bool) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string, io.Reader) error); ok { - r1 = rf(ctx, bucket, name, reader) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewObjectManager creates a new instance of ObjectManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewObjectManager(t interface { - mock.TestingT - Cleanup(func()) -}) *ObjectManager { - mock := &ObjectManager{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/ossfs/internal/utils/contract.go b/ossfs/internal/utils/contract.go deleted file mode 100644 index 9c458ec0..00000000 --- a/ossfs/internal/utils/contract.go +++ /dev/null @@ -1,21 +0,0 @@ -package utils - -import ( - "context" - "io" - "os" -) - -type CleanUp func() - -type ObjectManager interface { - GetObject(ctx context.Context, bucket, name string) (io.Reader, CleanUp, error) - GetObjectPart(ctx context.Context, bucket, name string, start, end int64) (io.Reader, CleanUp, error) - DeleteObject(ctx context.Context, bucket, name string) error - IsObjectExist(ctx context.Context, bucket, name string) (bool, error) - PutObject(ctx context.Context, bucket, name string, reader io.Reader) (bool, error) - CopyObject(ctx context.Context, bucket, srcName, targetName string) error - GetObjectMeta(ctx context.Context, bucket, name string) (os.FileInfo, error) - ListObjects(ctx context.Context, bucket, prefix string, count int) ([]os.FileInfo, error) - ListAllObjects(ctx context.Context, bucket, prefix string) ([]os.FileInfo, error) -} diff --git a/ossfs/internal/utils/init.go b/ossfs/internal/utils/init.go deleted file mode 100644 index 72709b88..00000000 --- a/ossfs/internal/utils/init.go +++ /dev/null @@ -1,6 +0,0 @@ -package utils - -func init() { - // Ensure OssObjectManager implements ObjectManager interface - var _ ObjectManager = (*OssObjectManager)(nil) -} diff --git a/ossfs/internal/utils/oss.go b/ossfs/internal/utils/oss.go deleted file mode 100644 index 6014d71d..00000000 --- a/ossfs/internal/utils/oss.go +++ /dev/null @@ -1,210 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "io" - "io/fs" - "os" - "strings" - "time" - - "github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss" - "github.com/spf13/afero" -) - -var ossDirSeparator string = "/" -var ossDefaultFileMode fs.FileMode = 0o755 - -type OssObjectManager struct { - ObjectManager - Client *oss.Client -} - -func (m *OssObjectManager) GetObject(ctx context.Context, bucket, name string) (io.Reader, CleanUp, error) { - req := &oss.GetObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(name), - } - res, err := m.Client.GetObject(ctx, req) - cleanUp := func() { - res.Body.Close() - } - return res.Body, cleanUp, err -} - -func (m *OssObjectManager) GetObjectPart(ctx context.Context, bucket, name string, start, end int64) (io.Reader, CleanUp, error) { - if start > end { - return nil, nil, afero.ErrOutOfRange - } - req := &oss.GetObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(name), - Range: oss.Ptr(fmt.Sprintf("bytes=%v-%v", start, end)), - RangeBehavior: oss.Ptr("standard"), - } - res, err := m.Client.GetObject(ctx, req) - cleanUp := func() { - res.Body.Close() - } - return res.Body, cleanUp, err -} - -func (m *OssObjectManager) DeleteObject(ctx context.Context, bucket, name string) error { - req := &oss.DeleteObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(name), - } - _, err := m.Client.DeleteObject(ctx, req) - return err -} - -func (m *OssObjectManager) IsObjectExist(ctx context.Context, bucket, name string) (bool, error) { - return m.Client.IsObjectExist(ctx, bucket, name) -} - -func (m *OssObjectManager) PutObject(ctx context.Context, bucket, name string, reader io.Reader) (bool, error) { - req := &oss.PutObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(name), - Body: reader, - } - _, err := m.Client.PutObject(ctx, req) - if err != nil { - return false, err - } - return true, nil -} - -func (m *OssObjectManager) CopyObject(ctx context.Context, bucket, srcName, targetName string) error { - req := &oss.CopyObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(srcName), - SourceKey: oss.Ptr(targetName), - SourceBucket: oss.Ptr(bucket), - StorageClass: oss.StorageClassStandard, - } - _, err := m.Client.CopyObject(ctx, req) - return err -} - -func (m *OssObjectManager) GetObjectMeta(ctx context.Context, bucket, name string) (os.FileInfo, error) { - req := &oss.HeadObjectRequest{ - Bucket: oss.Ptr(bucket), - Key: oss.Ptr(name), - } - - res, err := m.Client.HeadObject(ctx, req) - if err != nil { - return nil, err - } - return &OssObjectMeta{ - name: name, - size: res.ContentLength, - lastModifiedAt: *res.LastModified, - }, nil -} - -func (m *OssObjectManager) ListObjects(ctx context.Context, bucket, prefix string, count int) ([]os.FileInfo, error) { - req := &oss.ListObjectsV2Request{ - Bucket: oss.Ptr(bucket), - Delimiter: oss.Ptr(ossDirSeparator), - Prefix: oss.Ptr(prefix), - } - p := m.Client.NewListObjectsV2Paginator(req) - - s := make([]os.FileInfo, 0) - - var i int - -loop: - for p.HasNext() { - page, err := p.NextPage(ctx) - if err != nil { - return nil, err - } - - for _, obj := range page.Contents { - i++ - if i == count { - break loop - } - s = append(s, &OssObjectMeta{ - name: oss.ToString(obj.Key), - size: obj.Size, - lastModifiedAt: oss.ToTime(obj.LastModified), - }) - } - } - - return s, nil -} - -func (m *OssObjectManager) ListAllObjects(ctx context.Context, bucket, prefix string) ([]os.FileInfo, error) { - req := &oss.ListObjectsV2Request{ - Bucket: oss.Ptr(bucket), - Prefix: oss.Ptr(prefix), - } - p := m.Client.NewListObjectsV2Paginator(req) - - s := make([]os.FileInfo, 0) - - for p.HasNext() { - page, err := p.NextPage(ctx) - if err != nil { - return nil, err - } - - for _, obj := range page.Contents { - s = append(s, &OssObjectMeta{ - name: oss.ToString(obj.Key), - size: obj.Size, - lastModifiedAt: oss.ToTime(obj.LastModified), - }) - } - } - - return s, nil -} - -type OssObjectMeta struct { - os.FileInfo - name string - size int64 - lastModifiedAt time.Time -} - -func NewOssObjectMeta(name string, size int64, updatedAt time.Time) *OssObjectMeta { - return &OssObjectMeta{ - name: name, - size: size, - lastModifiedAt: updatedAt, - } -} - -func (objMeta *OssObjectMeta) isDir() bool { - return strings.HasSuffix(objMeta.name, ossDirSeparator) -} - -func (objMeta *OssObjectMeta) ModTime() time.Time { - return objMeta.lastModifiedAt -} - -func (objMeta *OssObjectMeta) Mode() fs.FileMode { - if objMeta.isDir() { - return ossDefaultFileMode | fs.ModeDir - } - return ossDefaultFileMode -} - -func (objMeta *OssObjectMeta) Name() string { - return objMeta.name -} - -func (objMeta *OssObjectMeta) Size() int64 { - return objMeta.size -} - -func (objMeta *OssObjectMeta) Sys() any { - return nil -}