cmd/sync: listall interface add to followlink parameters

cmd/destroy.go

@@ -158,7 +158,7 @@ func destroy(ctx *cli.Context) error {
 		}
 	}
 
-	objs, err := osync.ListAll(blob, "", "", "")
+	objs, err := osync.ListAll(blob, "", "", "", true)
 	if err != nil {
 		logger.Fatalf("list all objects: %s", err)
 	}

cmd/format.go

@@ -474,7 +474,7 @@ func format(c *cli.Context) error {
 			logger.Fatalf("Storage %s is not configured correctly: %s", blob, err)
 		}
 		if create {
-			if objs, err := osync.ListAll(blob, "", "", ""); err == nil {
+			if objs, err := osync.ListAll(blob, "", "", "", true); err == nil {
 				for o := range objs {
 					if o == nil {
 						logger.Warnf("List storage %s failed", blob)

cmd/fsck.go

@@ -107,7 +107,7 @@ func fsck(ctx *cli.Context) error {
 	}
 	logger.Infof("Data use %s", blob)
 	blob = object.WithPrefix(blob, "chunks/")
-	objs, err := osync.ListAll(blob, "", "", "")
+	objs, err := osync.ListAll(blob, "", "", "", true)
 	if err != nil {
 		logger.Fatalf("list all blocks: %s", err)
 	}

cmd/gc.go

@@ -232,7 +232,7 @@ func gc(ctx *cli.Context) error {
 
 	// Scan all objects to find leaked ones
 	blob = object.WithPrefix(blob, "chunks/")
-	objs, err := osync.ListAll(blob, "", "", "")
+	objs, err := osync.ListAll(blob, "", "", "", true)
 	if err != nil {
 		logger.Fatalf("list all blocks: %s", err)
 	}

cmd/objbench.go

@@ -569,7 +569,7 @@ func (bm *benchMarkObj) head(key string, startKey int) error {
 }
 
 func (bm *benchMarkObj) list(key string, startKey int) error {
-	result, err := osync.ListAll(bm.blob, "", "", "")
+	result, err := osync.ListAll(bm.blob, "", "", "", true)
 	for range result {
 	}
 	return err
@@ -588,7 +588,7 @@ func (bm *benchMarkObj) chtimes(key string, startKey int) error {
 }
 
 func listAll(s object.ObjectStorage, prefix, marker string, limit int64) ([]object.Object, error) {
-	ch, err := object.ListAll(s, prefix, marker)
+	ch, err := object.ListAll(s, prefix, marker, true)
 	if err == nil {
 		objs := make([]object.Object, 0)
 		for obj := range ch {
@@ -896,7 +896,7 @@ func functionalTesting(blob object.ObjectStorage, result *[][]string, colorful b
 		if err := blob.Put(key, bytes.NewReader([]byte("1"))); err != nil {
 			return fmt.Errorf("put encode file failed: %s", err)
 		} else {
-			if resp, err := blob.List("", "测试编码文件", "", 1); err != nil && err != utils.ENOTSUP {
+			if resp, err := blob.List("", "测试编码文件", "", 1, true); err != nil && err != utils.ENOTSUP {
 				return fmt.Errorf("list encode file failed %s", err)
 			} else if len(resp) == 1 && resp[0].Key() != key {
 				return fmt.Errorf("list encode file failed: expect key %s, but got %s", key, resp[0].Key())

cmd/object.go

@@ -200,7 +200,7 @@ func (j *juiceFS) Head(key string) (object.Object, error) {
 	return &jObj{key, fi}, nil
 }
 
-func (j *juiceFS) List(prefix, marker, delimiter string, limit int64) ([]object.Object, error) {
+func (j *juiceFS) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]object.Object, error) {
 	if delimiter != "/" {
 		return nil, utils.ENOTSUP
 	}
@@ -221,7 +221,7 @@ func (j *juiceFS) List(prefix, marker, delimiter string, limit int64) ([]object.
 		}
 		objs = append(objs, obj)
 	}
-	entries, err := j.readDirSorted(dir)
+	entries, err := j.readDirSorted(dir, followLink)
 	if err != 0 {
 		if err == syscall.ENOENT {
 			return nil, nil
@@ -250,7 +250,7 @@ type mEntry struct {
 
 // readDirSorted reads the directory named by dirname and returns
 // a sorted list of directory entries.
-func (j *juiceFS) readDirSorted(dirname string) ([]*mEntry, syscall.Errno) {
+func (j *juiceFS) readDirSorted(dirname string, followLink bool) ([]*mEntry, syscall.Errno) {
 	f, err := j.jfs.Open(ctx, dirname, 0)
 	if err != 0 {
 		return nil, err
@@ -265,8 +265,7 @@ func (j *juiceFS) readDirSorted(dirname string) ([]*mEntry, syscall.Errno) {
 		fi := fs.AttrToFileInfo(e.Inode, e.Attr)
 		if fi.IsDir() {
 			mEntries[i] = &mEntry{fi, string(e.Name) + dirSuffix, false}
-		} else if fi.IsSymlink() {
-			// follow symlink
+		} else if fi.IsSymlink() && followLink {
 			fi2, err := j.jfs.Stat(ctx, filepath.Join(dirname, string(e.Name)))
 			if err != 0 {
 				mEntries[i] = &mEntry{fi, string(e.Name), true}
@@ -276,9 +275,9 @@ func (j *juiceFS) readDirSorted(dirname string) ([]*mEntry, syscall.Errno) {
 			if fi2.IsDir() {
 				name += dirSuffix
 			}
-			mEntries[i] = &mEntry{fi2, name, true}
+			mEntries[i] = &mEntry{fi2, name, false}
 		} else {
-			mEntries[i] = &mEntry{fi, string(e.Name), false}
+			mEntries[i] = &mEntry{fi, string(e.Name), fi.IsSymlink()}
 		}
 	}
 	sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].name < mEntries[j].name })

pkg/object/azure.go

@@ -132,7 +132,7 @@ func (b *wasb) Delete(key string) error {
 	return err
 }
 
-func (b *wasb) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+func (b *wasb) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
 	if delimiter != "" {
 		return nil, notSupported
 	}

pkg/object/b2.go

@@ -122,7 +122,7 @@ func (c *b2client) Delete(key string) error {
 	return err
 }
 
-func (c *b2client) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+func (c *b2client) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
 	if limit > 1000 {
 		limit = 1000
 	}

pkg/object/bos.go

@@ -138,7 +138,7 @@ func (q *bosclient) Delete(key string) error {
 	return err
 }
 
-func (q *bosclient) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+func (q *bosclient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
 	if limit > 1000 {
 		limit = 1000
 	}

pkg/object/ceph.go

@@ -203,7 +203,7 @@ func (c *ceph) Head(key string) (Object, error) {
 	return o, err
 }
 
-func (c *ceph) ListAll(prefix, marker string) (<-chan Object, error) {
+func (c *ceph) ListAll(prefix, marker string, followLink bool) (<-chan Object, error) {
 	var objs = make(chan Object, 1000)
 	err := c.do(func(ctx *rados.IOContext) error {
 		iter, err := ctx.Iter()

pkg/object/cos.go

@@ -155,7 +155,7 @@ func (c *COS) Delete(key string) error {
 	return err
 }
 
-func (c *COS) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+func (c *COS) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
 	param := cos.BucketGetOptions{
 		Prefix:       prefix,
 		Marker:       marker,
@@ -197,7 +197,7 @@ func (c *COS) List(prefix, marker, delimiter string, limit int64) ([]Object, err
 	return objs, nil
 }
 
-func (c *COS) ListAll(prefix, marker string) (<-chan Object, error) {
+func (c *COS) ListAll(prefix, marker string, followLink bool) (<-chan Object, error) {
 	return nil, notSupported
 }
 

pkg/object/etcd.go

@@ -111,7 +111,7 @@ func genNextKey(key string) string {
 	return string(next)
 }
 
-func (c *etcdClient) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+func (c *etcdClient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
 	if delimiter != "" {
 		return nil, notSupported
 	}

pkg/object/file.go

@@ -224,7 +224,7 @@ func (m *mEntry) IsDir() bool {
 
 // readDirSorted reads the directory named by dirname and returns
 // a sorted list of directory entries.
-func readDirSorted(dirname string) ([]*mEntry, error) {
+func readDirSorted(dirname string, followLink bool) ([]*mEntry, error) {
 	f, err := os.Open(dirname)
 	if err != nil {
 		return nil, err
@@ -239,8 +239,7 @@ func readDirSorted(dirname string) ([]*mEntry, error) {
 	for i, e := range entries {
 		if e.IsDir() {
 			mEntries[i] = &mEntry{e, e.Name() + dirSuffix, nil, false}
-		} else if !e.Type().IsRegular() {
-			// follow symlink
+		} else if !e.Type().IsRegular() && followLink {
 			fi, err := os.Stat(filepath.Join(dirname, e.Name()))
 			if err != nil {
 				mEntries[i] = &mEntry{e, e.Name(), nil, true}
@@ -250,16 +249,16 @@ func readDirSorted(dirname string) ([]*mEntry, error) {
 			if fi.IsDir() {
 				name = e.Name() + dirSuffix
 			}
-			mEntries[i] = &mEntry{e, name, fi, true}
+			mEntries[i] = &mEntry{e, name, fi, false}
 		} else {
-			mEntries[i] = &mEntry{e, e.Name(), nil, false}
+			mEntries[i] = &mEntry{e, e.Name(), nil, !e.Type().IsRegular()}
 		}
 	}
 	sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })
 	return mEntries, err
 }
 
-func (d *filestore) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+func (d *filestore) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
 	if delimiter != "/" {
 		return nil, notSupported
 	}
@@ -280,7 +279,7 @@ func (d *filestore) List(prefix, marker, delimiter string, limit int64) ([]Objec
 		}
 		objs = append(objs, obj)
 	}
-	entries, err := readDirSorted(dir)
+	entries, err := readDirSorted(dir, followLink)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, nil

pkg/object/filesystem_test.go

@@ -92,7 +92,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
 	// cleanup
 	defer func() {
 		// delete reversely, directory only can be deleted when it's empty
-		objs, err := listAll(s, "", "", 100)
+		objs, err := listAll(s, "", "", 100, true)
 		if err != nil {
 			t.Fatalf("listall failed: %s", err)
 		}
@@ -107,7 +107,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
 			}
 		}
 	}()
-	objs, err := listAll(s, "x/", "", 100)
+	objs, err := listAll(s, "x/", "", 100, true)
 	if err != nil {
 		t.Fatalf("list failed: %s", err)
 	}
@@ -116,7 +116,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
 		t.Fatalf("testKeysEqual fail: %s", err)
 	}
 
-	objs, err = listAll(s, "x", "", 100)
+	objs, err = listAll(s, "x", "", 100, true)
 	if err != nil {
 		t.Fatalf("list failed: %s", err)
 	}
@@ -125,7 +125,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
 		t.Fatalf("testKeysEqual fail: %s", err)
 	}
 
-	objs, err = listAll(s, "xy", "", 100)
+	objs, err = listAll(s, "xy", "", 100, true)
 	if err != nil {
 		t.Fatalf("list failed: %s", err)
 	}
@@ -159,7 +159,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
 		}
 		_ = ss.Symlink("xyz/notExist/", "b")
 
-		objs, err = listAll(s, "", "", 100)
+		objs, err = listAll(s, "", "", 100, true)
 		if err != nil {
 			t.Fatalf("listall failed: %s", err)
 		}
@@ -173,5 +173,14 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
 		if objs[9].Size() != 10 {
 			t.Fatalf("size of target(file) should be 10")
 		}
+
+		// test don't follow symlink
+		if _, ok := s.(*hdfsclient); !ok {
+			objs, err = listAll(s, "", "", 100, false)
+			expectedKeys = []string{"", "a", "a-", "a0", "b", "b-", "b0", "bb/", "bb/b1", "x/", "x/x.txt", "xy.txt", "xyz/", "xyz/ol1/", "xyz/ol1/p.txt", "xyz/xyz.txt"}
+			if err = testKeysEqual(objs, expectedKeys); err != nil {
+				t.Fatalf("testKeysEqual fail: %s", err)
+			}
+		}
 	}
 }

pkg/object/gluster.go

@@ -147,7 +147,7 @@ func (c *gluster) Delete(key string) error {
 
 // readDirSorted reads the directory named by dirname and returns
 // a sorted list of directory entries.
-func (d *gluster) readDirSorted(dirname string) ([]*mEntry, error) {
+func (d *gluster) readDirSorted(dirname string, followLink bool) ([]*mEntry, error) {
 	f, err := d.vol.Open(dirname)
 	if err != nil {
 		return nil, err
@@ -166,8 +166,7 @@ func (d *gluster) readDirSorted(dirname string) ([]*mEntry, error) {
 		}
 		if e.IsDir() {
 			mEntries = append(mEntries, &mEntry{nil, name + dirSuffix, e, false})
-		} else if !e.Mode().IsRegular() {
-			// follow symlink
+		} else if !e.Mode().IsRegular() && followLink {
 			fi, err := d.vol.Stat(filepath.Join(dirname, name))
 			if err != nil {
 				mEntries = append(mEntries, &mEntry{nil, name, e, true})
@@ -176,16 +175,16 @@ func (d *gluster) readDirSorted(dirname string) ([]*mEntry, error) {
 			if fi.IsDir() {
 				name += dirSuffix
 			}
-			mEntries = append(mEntries, &mEntry{nil, name, fi, true})
+			mEntries = append(mEntries, &mEntry{nil, name, fi, false})
 		} else {
-			mEntries = append(mEntries, &mEntry{nil, name, e, false})
+			mEntries = append(mEntries, &mEntry{nil, name, e, !e.Mode().IsRegular()})
 		}
 	}
 	sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })
 	return mEntries, err
 }
 
-func (d *gluster) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+func (d *gluster) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
 	if delimiter != "/" {
 		return nil, notSupported
 	}
@@ -206,7 +205,7 @@ func (d *gluster) List(prefix, marker, delimiter string, limit int64) ([]Object,
 		}
 		objs = append(objs, obj)
 	}
-	entries, err := d.readDirSorted(dir)
+	entries, err := d.readDirSorted(dir, followLink)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, nil

pkg/object/gs.go

@@ -51,7 +51,7 @@ func (g *gs) String() string {
 
 func (g *gs) Create() error {
 	// check if the bucket is already exists
-	if objs, err := g.List("", "", "", 1); err == nil && len(objs) > 0 {
+	if objs, err := g.List("", "", "", 1, true); err == nil && len(objs) > 0 {
 		return nil
 	}
 
@@ -141,7 +141,7 @@ func (g *gs) Delete(key string) error {
 	return nil
 }
 
-func (g *gs) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+func (g *gs) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
 	if marker != "" && g.pageToken == "" {
 		// last page
 		return nil, nil

pkg/object/hdfs.go

@@ -176,7 +176,7 @@ func (h *hdfsclient) Delete(key string) error {
 	return err
 }
 
-func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
 	if delimiter != "/" {
 		return nil, notSupported
 	}

pkg/object/ibmcos.go

@@ -165,7 +165,7 @@ func (s *ibmcos) Delete(key string) error {
 	return err
 }
 
-func (s *ibmcos) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
+func (s *ibmcos) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
 	param := s3.ListObjectsInput{
 		Bucket:       &s.bucket,
 		Prefix:       &prefix,
@@ -203,7 +203,7 @@ func (s *ibmcos) List(prefix, marker, delimiter string, limit int64) ([]Object,
 	return objs, nil
 }
 
-func (s *ibmcos) ListAll(prefix, marker string) (<-chan Object, error) {
+func (s *ibmcos) ListAll(prefix, marker string, followLink bool) (<-chan Object, error) {
 	return nil, notSupported
 }
 

pkg/object/interface.go

@@ -92,9 +92,9 @@ type ObjectStorage interface {
 	// Head returns some information about the object or an error if not found.
 	Head(key string) (Object, error)
 	// List returns a list of objects.
-	List(prefix, marker, delimiter string, limit int64) ([]Object, error)
+	List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error)
 	// ListAll returns all the objects as an channel.
-	ListAll(prefix, marker string) (<-chan Object, error)
+	ListAll(prefix, marker string, followLink bool) (<-chan Object, error)
 
 	// CreateMultipartUpload starts to upload a large object part by part.
 	CreateMultipartUpload(key string) (*MultipartUpload, error)