gateway: optimize list performance (#4969)

go.mod

@@ -253,7 +253,7 @@ require (
 	xorm.io/builder v0.3.7 // indirect
 )
 
-replace github.com/minio/minio v0.0.0-20210206053228-97fe57bba92c => github.com/juicedata/minio v0.0.0-20240523035739-2e3d9abd6140
+replace github.com/minio/minio v0.0.0-20210206053228-97fe57bba92c => github.com/juicedata/minio v0.0.0-20240612084555-8a33dba0f571
 
 replace github.com/hanwen/go-fuse/v2 v2.1.1-0.20210611132105-24a1dfe6b4f8 => github.com/juicedata/go-fuse/v2 v2.1.1-0.20240425033113-7c40cb5eb3e9
 

go.sum

@@ -511,8 +511,8 @@ github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825 h1:7KrwI4HPqvNLK
 github.com/juicedata/gogfapi v0.0.0-20230626071140-fc28e5537825/go.mod h1:Ho5G4KgrgbMKW0buAJdOmYoJcOImkzznJQaLiATrsx4=
 github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible h1:2/ttSmYoX+QMegpNyAJR0Y6aHcVk57F7RJit5xN2T/s=
 github.com/juicedata/huaweicloud-sdk-go-obs v3.22.12-0.20230228031208-386e87b5c091+incompatible/go.mod h1:Ukwa8ffRQLV6QRwpqGioPjn2Wnf7TBDA4DbennDOqHE=
-github.com/juicedata/minio v0.0.0-20240523035739-2e3d9abd6140 h1:/ohtUZCmcCfUDOwjJ8B921fHenbV8JJvNt9oxDDN5/0=
-github.com/juicedata/minio v0.0.0-20240523035739-2e3d9abd6140/go.mod h1:UOWyfa3ls1tnpJrNw2yzGqfrwM4nzsZq/qz+zd6H+/Q=
+github.com/juicedata/minio v0.0.0-20240612084555-8a33dba0f571 h1:usZ1RJ9AssiJxSVRibh2mbQ5cI9FuSbkXJTiweL3hxk=
+github.com/juicedata/minio v0.0.0-20240612084555-8a33dba0f571/go.mod h1:UOWyfa3ls1tnpJrNw2yzGqfrwM4nzsZq/qz+zd6H+/Q=
 github.com/juicedata/mpb/v7 v7.0.4-0.20231024073412-2b8d31be510b h1:0/6suPNZnrOlRlBaU/Bnitu8HiKkkLSzQhHbwQ9AysM=
 github.com/juicedata/mpb/v7 v7.0.4-0.20231024073412-2b8d31be510b/go.mod h1:NXGsfPGx6G2JssqvEcULtDqUrxuuYs4llpv8W6ZUpzk=
 github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI=

pkg/gateway/gateway.go

@@ -312,14 +312,14 @@ func (n *jfsObjects) isLeaf(bucket, leafPath string) bool {
 }
 
 func (n *jfsObjects) listDirFactory() minio.ListDirFunc {
-	return func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
+	return func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []*minio.Entry, delayIsLeaf bool) {
 		f, eno := n.fs.Open(mctx, n.path(bucket, prefixDir), 0)
 		if eno != 0 {
 			return fs.IsNotExist(eno), nil, false
 		}
 		defer f.Close(mctx)
 		if fi, _ := f.Stat(); fi.(*fs.FileStat).Atime() == 0 && prefixEntry == "" {
-			entries = append(entries, "")
+			entries = append(entries, &minio.Entry{Name: ""})
 		}
 
 		fis, eno := f.Readdir(mctx, 0)
@@ -331,6 +331,16 @@ func (n *jfsObjects) listDirFactory() minio.ListDirFunc {
 			if root && (fi.Name() == metaBucket || fi.Name() == minio.MinioMetaBucket) {
 				continue
 			}
+			entry := &minio.Entry{Name: fi.Name(),
+				Info: &minio.ObjectInfo{
+					Bucket:  bucket,
+					Name:    fi.Name(),
+					ModTime: fi.ModTime(),
+					Size:    fi.Size(),
+					IsDir:   fi.IsDir(),
+					AccTime: fi.ModTime(),
+				},
+			}
 			if stat, ok := fi.(*fs.FileStat); ok && stat.IsSymlink() {
 				var err syscall.Errno
 				p := n.path(bucket, prefixDir, fi.Name())
@@ -340,10 +350,10 @@ func (n *jfsObjects) listDirFactory() minio.ListDirFunc {
 				}
 			}
 			if fi.IsDir() {
-				entries = append(entries, fi.Name()+sep)
-			} else {
-				entries = append(entries, fi.Name())
+				entry.Name += sep
+				entry.Info.Size = 0
 			}
+			entries = append(entries, entry)
 		}
 		if len(entries) == 0 {
 			return true, nil, false
@@ -368,42 +378,47 @@ func (n *jfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 	if err := n.checkBucket(ctx, bucket); err != nil {
 		return loi, err
 	}
-	getObjectInfo := func(ctx context.Context, bucket, object string) (obj minio.ObjectInfo, err error) {
+	getObjectInfo := func(ctx context.Context, bucket, object string, info *minio.ObjectInfo) (obj minio.ObjectInfo, err error) {
+		if info != nil && info.Size > 0 {
+			info.Name = object
+			return *info, nil
+		}
 		fi, eno := n.fs.Stat(mctx, n.path(bucket, object))
 		if eno == 0 {
-			var etag []byte
-			if n.gConf.KeepEtag && !fi.IsDir() {
-				etag, _ = n.fs.GetXattr(mctx, n.path(bucket, object), s3Etag)
-			}
 			size := fi.Size()
 			if fi.IsDir() {
 				size = 0
 			}
-			obj = minio.ObjectInfo{
+			info = &minio.ObjectInfo{
 				Bucket:  bucket,
-				Name:    object,
 				ModTime: fi.ModTime(),
 				Size:    size,
 				IsDir:   fi.IsDir(),
 				AccTime: fi.ModTime(),
-				ETag:    string(etag),
 			}
 		}
 
 		// replace links to external file systems with empty files
 		if eno == syscall.ENOTSUP {
 			now := time.Now()
-			obj = minio.ObjectInfo{
+			info = &minio.ObjectInfo{
 				Bucket:  bucket,
-				Name:    object,
 				ModTime: now,
 				Size:    0,
 				IsDir:   false,
 				AccTime: now,
 			}
 			eno = 0
 		}
-		return obj, jfsToObjectErr(ctx, eno, bucket, object)
+		if info == nil {
+			return obj, jfsToObjectErr(ctx, eno, bucket, object)
+		}
+		info.Name = object
+		if n.gConf.KeepEtag && !strings.HasSuffix(object, sep) {
+			etag, _ := n.fs.GetXattr(mctx, n.path(bucket, object), s3Etag)
+			info.ETag = string(etag)
+		}
+		return *info, jfsToObjectErr(ctx, eno, bucket, object)
 	}
 
 	if maxKeys == 0 {
@@ -1304,7 +1319,7 @@ func (n *jfsObjects) ListObjectVersions(ctx context.Context, bucket, prefix, mar
 	return loi, minio.NotImplemented{}
 }
 
-func (n *jfsObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string) (oi minio.ObjectInfo, e error) {
+func (n *jfsObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string, info *minio.ObjectInfo) (oi minio.ObjectInfo, e error) {
 	return n.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }