Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gateway: optimize list performance #4969

Merged
merged 3 commits into from
Jun 28, 2024
Merged

gateway: optimize list performance #4969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e65a663
gateway: optimize list performance
zhijian-pro Jun 24, 2024
7be0018
fix golangci lint
zhijian-pro Jun 24, 2024
2d3b13b
fix
zhijian-pro Jun 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ require (
xorm.io/builder v0.3.7 // indirect
)

replace github.com/minio/minio v0.0.0-20210206053228-97fe57bba92c => github.com/juicedata/minio v0.0.0-20240523035739-2e3d9abd6140
replace github.com/minio/minio v0.0.0-20210206053228-97fe57bba92c => github.com/juicedata/minio v0.0.0-20240612084555-8a33dba0f571

replace github.com/hanwen/go-fuse/v2 v2.1.1-0.20210611132105-24a1dfe6b4f8 => github.com/juicedata/go-fuse/v2 v2.1.1-0.20240425033113-7c40cb5eb3e9

Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -511,8 +511,8 @@ github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 h1:7KrwI4HPqvNLK
github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4=
github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s=
github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible/go.mod h1:Ukwa8ffRQLV6QRwpqGioPjn2Wnf7TBDA4DbennDOqHE=
github.com/juicedata/minio v0.0.0-20240523035739-2e3d9abd6140 h1:/ohtUZCmcCfUDOwjJ8B921fHenbV8JJvNt9oxDDN5/0=
github.com/juicedata/minio v0.0.0-20240523035739-2e3d9abd6140/go.mod h1:UOWyfa3ls1tnpJrNw2yzGqfrwM4nzsZq/qz+zd6H+/Q=
github.com/juicedata/minio v0.0.0-20240612084555-8a33dba0f571 h1:usZ1RJ9AssiJxSVRibh2mbQ5cI9FuSbkXJTiweL3hxk=
github.com/juicedata/minio v0.0.0-20240612084555-8a33dba0f571/go.mod h1:UOWyfa3ls1tnpJrNw2yzGqfrwM4nzsZq/qz+zd6H+/Q=
github.com/juicedata/mpb/v7 v7.0.4-0.20231024073412-2b8d31be510b h1:0/6suPNZnrOlRlBaU/Bnitu8HiKkkLSzQhHbwQ9AysM=
github.com/juicedata/mpb/v7 v7.0.4-0.20231024073412-2b8d31be510b/go.mod h1:NXGsfPGx6G2JssqvEcULtDqUrxuuYs4llpv8W6ZUpzk=
github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI=
Expand Down
49 changes: 32 additions & 17 deletions pkg/gateway/gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,14 +312,14 @@ func (n *jfsObjects) isLeaf(bucket, leafPath string) bool {
}

func (n *jfsObjects) listDirFactory() minio.ListDirFunc {
return func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
return func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []*minio.Entry, delayIsLeaf bool) {
f, eno := n.fs.Open(mctx, n.path(bucket, prefixDir), 0)
if eno != 0 {
return fs.IsNotExist(eno), nil, false
}
defer f.Close(mctx)
if fi, _ := f.Stat(); fi.(*fs.FileStat).Atime() == 0 && prefixEntry == "" {
entries = append(entries, "")
entries = append(entries, &minio.Entry{Name: ""})
}

fis, eno := f.Readdir(mctx, 0)
Expand All @@ -331,6 +331,16 @@ func (n *jfsObjects) listDirFactory() minio.ListDirFunc {
if root && (fi.Name() == metaBucket || fi.Name() == minio.MinioMetaBucket) {
continue
}
entry := &minio.Entry{Name: fi.Name(),
Info: &minio.ObjectInfo{
Bucket: bucket,
Name: fi.Name(),
ModTime: fi.ModTime(),
Size: fi.Size(),
IsDir: fi.IsDir(),
AccTime: fi.ModTime(),
},
}
if stat, ok := fi.(*fs.FileStat); ok && stat.IsSymlink() {
var err syscall.Errno
p := n.path(bucket, prefixDir, fi.Name())
Expand All @@ -340,10 +350,10 @@ func (n *jfsObjects) listDirFactory() minio.ListDirFunc {
}
}
if fi.IsDir() {
entries = append(entries, fi.Name()+sep)
} else {
entries = append(entries, fi.Name())
entry.Name += sep
entry.Info.Size = 0
}
entries = append(entries, entry)
}
if len(entries) == 0 {
return true, nil, false
Expand All @@ -368,42 +378,47 @@ func (n *jfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
if err := n.checkBucket(ctx, bucket); err != nil {
return loi, err
}
getObjectInfo := func(ctx context.Context, bucket, object string) (obj minio.ObjectInfo, err error) {
getObjectInfo := func(ctx context.Context, bucket, object string, info *minio.ObjectInfo) (obj minio.ObjectInfo, err error) {
if info != nil && info.Size > 0 {
info.Name = object
return *info, nil
}
fi, eno := n.fs.Stat(mctx, n.path(bucket, object))
if eno == 0 {
var etag []byte
if n.gConf.KeepEtag && !fi.IsDir() {
etag, _ = n.fs.GetXattr(mctx, n.path(bucket, object), s3Etag)
}
size := fi.Size()
if fi.IsDir() {
size = 0
}
obj = minio.ObjectInfo{
info = &minio.ObjectInfo{
Bucket: bucket,
Name: object,
ModTime: fi.ModTime(),
Size: size,
IsDir: fi.IsDir(),
AccTime: fi.ModTime(),
ETag: string(etag),
}
}

// replace links to external file systems with empty files
if eno == syscall.ENOTSUP {
now := time.Now()
obj = minio.ObjectInfo{
info = &minio.ObjectInfo{
Bucket: bucket,
Name: object,
ModTime: now,
Size: 0,
IsDir: false,
AccTime: now,
}
eno = 0
}
return obj, jfsToObjectErr(ctx, eno, bucket, object)
if info == nil {
return obj, jfsToObjectErr(ctx, eno, bucket, object)
}
info.Name = object
if n.gConf.KeepEtag && !strings.HasSuffix(object, sep) {
etag, _ := n.fs.GetXattr(mctx, n.path(bucket, object), s3Etag)
info.ETag = string(etag)
}
return *info, jfsToObjectErr(ctx, eno, bucket, object)
}

if maxKeys == 0 {
Expand Down Expand Up @@ -1304,7 +1319,7 @@ func (n *jfsObjects) ListObjectVersions(ctx context.Context, bucket, prefix, mar
return loi, minio.NotImplemented{}
}

func (n *jfsObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string) (oi minio.ObjectInfo, e error) {
func (n *jfsObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string, info *minio.ObjectInfo) (oi minio.ObjectInfo, e error) {
return n.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
}

Expand Down
Loading