Skip to content

Commit

Permalink
cmd/sync: listall interface add to followlink parameters (#3942)
Browse files Browse the repository at this point in the history 
close #3934

---------

Co-authored-by: Davies Liu <[email protected]>
  • Loading branch information
@zhijian-pro @davies
zhijian-pro and davies authored Aug 1, 2023
1 parent 62828d4 commit 66bdd05
Show file tree
Hide file tree
Showing 47 changed files with 138 additions and 131 deletions.
2 changes: 1 addition & 1 deletion cmd/destroy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ func destroy(ctx *cli.Context) error {
}
}

objs, err := osync.ListAll(blob, "", "", "")
objs, err := osync.ListAll(blob, "", "", "", true)
if err != nil {
logger.Fatalf("list all objects: %s", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/format.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,7 +474,7 @@ func format(c *cli.Context) error {
logger.Fatalf("Storage %s is not configured correctly: %s", blob, err)
}
if create {
if objs, err := osync.ListAll(blob, "", "", ""); err == nil {
if objs, err := osync.ListAll(blob, "", "", "", true); err == nil {
for o := range objs {
if o == nil {
logger.Warnf("List storage %s failed", blob)
Expand Down
2 changes: 1 addition & 1 deletion cmd/fsck.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func fsck(ctx *cli.Context) error {
}
logger.Infof("Data use %s", blob)
blob = object.WithPrefix(blob, "chunks/")
objs, err := osync.ListAll(blob, "", "", "")
objs, err := osync.ListAll(blob, "", "", "", true)
if err != nil {
logger.Fatalf("list all blocks: %s", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/gc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ func gc(ctx *cli.Context) error {

// Scan all objects to find leaked ones
blob = object.WithPrefix(blob, "chunks/")
objs, err := osync.ListAll(blob, "", "", "")
objs, err := osync.ListAll(blob, "", "", "", true)
if err != nil {
logger.Fatalf("list all blocks: %s", err)
}
Expand Down
6 changes: 3 additions & 3 deletions cmd/objbench.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,7 @@ func (bm *benchMarkObj) head(key string, startKey int) error {
}

func (bm *benchMarkObj) list(key string, startKey int) error {
result, err := osync.ListAll(bm.blob, "", "", "")
result, err := osync.ListAll(bm.blob, "", "", "", true)
for range result {
}
return err
Expand All @@ -588,7 +588,7 @@ func (bm *benchMarkObj) chtimes(key string, startKey int) error {
}

func listAll(s object.ObjectStorage, prefix, marker string, limit int64) ([]object.Object, error) {
ch, err := object.ListAll(s, prefix, marker)
ch, err := object.ListAll(s, prefix, marker, true)
if err == nil {
objs := make([]object.Object, 0)
for obj := range ch {
Expand Down Expand Up @@ -896,7 +896,7 @@ func functionalTesting(blob object.ObjectStorage, result *[][]string, colorful b
if err := blob.Put(key, bytes.NewReader([]byte("1"))); err != nil {
return fmt.Errorf("put encode file failed: %s", err)
} else {
if resp, err := blob.List("", "测试编码文件", "", 1); err != nil && err != utils.ENOTSUP {
if resp, err := blob.List("", "测试编码文件", "", 1, true); err != nil && err != utils.ENOTSUP {
return fmt.Errorf("list encode file failed %s", err)
} else if len(resp) == 1 && resp[0].Key() != key {
return fmt.Errorf("list encode file failed: expect key %s, but got %s", key, resp[0].Key())
Expand Down
13 changes: 6 additions & 7 deletions cmd/object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ func (j *juiceFS) Head(key string) (object.Object, error) {
return &jObj{key, fi}, nil
}

func (j *juiceFS) List(prefix, marker, delimiter string, limit int64) ([]object.Object, error) {
func (j *juiceFS) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]object.Object, error) {
if delimiter != "/" {
return nil, utils.ENOTSUP
}
Expand All @@ -221,7 +221,7 @@ func (j *juiceFS) List(prefix, marker, delimiter string, limit int64) ([]object.
}
objs = append(objs, obj)
}
entries, err := j.readDirSorted(dir)
entries, err := j.readDirSorted(dir, followLink)
if err != 0 {
if err == syscall.ENOENT {
return nil, nil
Expand Down Expand Up @@ -250,7 +250,7 @@ type mEntry struct {

// readDirSorted reads the directory named by dirname and returns
// a sorted list of directory entries.
func (j *juiceFS) readDirSorted(dirname string) ([]*mEntry, syscall.Errno) {
func (j *juiceFS) readDirSorted(dirname string, followLink bool) ([]*mEntry, syscall.Errno) {
f, err := j.jfs.Open(ctx, dirname, 0)
if err != 0 {
return nil, err
Expand All @@ -265,8 +265,7 @@ func (j *juiceFS) readDirSorted(dirname string) ([]*mEntry, syscall.Errno) {
fi := fs.AttrToFileInfo(e.Inode, e.Attr)
if fi.IsDir() {
mEntries[i] = &mEntry{fi, string(e.Name) + dirSuffix, false}
} else if fi.IsSymlink() {
// follow symlink
} else if fi.IsSymlink() && followLink {
fi2, err := j.jfs.Stat(ctx, filepath.Join(dirname, string(e.Name)))
if err != 0 {
mEntries[i] = &mEntry{fi, string(e.Name), true}
Expand All @@ -276,9 +275,9 @@ func (j *juiceFS) readDirSorted(dirname string) ([]*mEntry, syscall.Errno) {
if fi2.IsDir() {
name += dirSuffix
}
mEntries[i] = &mEntry{fi2, name, true}
mEntries[i] = &mEntry{fi2, name, false}
} else {
mEntries[i] = &mEntry{fi, string(e.Name), false}
mEntries[i] = &mEntry{fi, string(e.Name), fi.IsSymlink()}
}
}
sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].name < mEntries[j].name })
Expand Down
2 changes: 1 addition & 1 deletion pkg/object/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func (b *wasb) Delete(key string) error {
return err
}

func (b *wasb) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
func (b *wasb) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
if delimiter != "" {
return nil, notSupported
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/object/b2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func (c *b2client) Delete(key string) error {
return err
}

func (c *b2client) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
func (c *b2client) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
if limit > 1000 {
limit = 1000
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/object/bos.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ func (q *bosclient) Delete(key string) error {
return err
}

func (q *bosclient) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
func (q *bosclient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
if limit > 1000 {
limit = 1000
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/object/ceph.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ func (c *ceph) Head(key string) (Object, error) {
return o, err
}

func (c *ceph) ListAll(prefix, marker string) (<-chan Object, error) {
func (c *ceph) ListAll(prefix, marker string, followLink bool) (<-chan Object, error) {
var objs = make(chan Object, 1000)
err := c.do(func(ctx *rados.IOContext) error {
iter, err := ctx.Iter()
Expand Down
4 changes: 2 additions & 2 deletions pkg/object/cos.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ func (c *COS) Delete(key string) error {
return err
}

func (c *COS) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
func (c *COS) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
param := cos.BucketGetOptions{
Prefix: prefix,
Marker: marker,
Expand Down Expand Up @@ -197,7 +197,7 @@ func (c *COS) List(prefix, marker, delimiter string, limit int64) ([]Object, err
return objs, nil
}

func (c *COS) ListAll(prefix, marker string) (<-chan Object, error) {
func (c *COS) ListAll(prefix, marker string, followLink bool) (<-chan Object, error) {
return nil, notSupported
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/object/etcd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func genNextKey(key string) string {
return string(next)
}

func (c *etcdClient) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
func (c *etcdClient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
if delimiter != "" {
return nil, notSupported
}
Expand Down
13 changes: 6 additions & 7 deletions pkg/object/file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ func (m *mEntry) IsDir() bool {

// readDirSorted reads the directory named by dirname and returns
// a sorted list of directory entries.
func readDirSorted(dirname string) ([]*mEntry, error) {
func readDirSorted(dirname string, followLink bool) ([]*mEntry, error) {
f, err := os.Open(dirname)
if err != nil {
return nil, err
Expand All @@ -239,8 +239,7 @@ func readDirSorted(dirname string) ([]*mEntry, error) {
for i, e := range entries {
if e.IsDir() {
mEntries[i] = &mEntry{e, e.Name() + dirSuffix, nil, false}
} else if !e.Type().IsRegular() {
// follow symlink
} else if !e.Type().IsRegular() && followLink {
fi, err := os.Stat(filepath.Join(dirname, e.Name()))
if err != nil {
mEntries[i] = &mEntry{e, e.Name(), nil, true}
Expand All @@ -250,16 +249,16 @@ func readDirSorted(dirname string) ([]*mEntry, error) {
if fi.IsDir() {
name = e.Name() + dirSuffix
}
mEntries[i] = &mEntry{e, name, fi, true}
mEntries[i] = &mEntry{e, name, fi, false}
} else {
mEntries[i] = &mEntry{e, e.Name(), nil, false}
mEntries[i] = &mEntry{e, e.Name(), nil, !e.Type().IsRegular()}
}
}
sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })
return mEntries, err
}

func (d *filestore) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
func (d *filestore) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
if delimiter != "/" {
return nil, notSupported
}
Expand All @@ -280,7 +279,7 @@ func (d *filestore) List(prefix, marker, delimiter string, limit int64) ([]Objec
}
objs = append(objs, obj)
}
entries, err := readDirSorted(dir)
entries, err := readDirSorted(dir, followLink)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
Expand Down
19 changes: 14 additions & 5 deletions pkg/object/filesystem_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
// cleanup
defer func() {
// delete reversely, directory only can be deleted when it's empty
objs, err := listAll(s, "", "", 100)
objs, err := listAll(s, "", "", 100, true)
if err != nil {
t.Fatalf("listall failed: %s", err)
}
Expand All @@ -107,7 +107,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
}
}
}()
objs, err := listAll(s, "x/", "", 100)
objs, err := listAll(s, "x/", "", 100, true)
if err != nil {
t.Fatalf("list failed: %s", err)
}
Expand All @@ -116,7 +116,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
t.Fatalf("testKeysEqual fail: %s", err)
}

objs, err = listAll(s, "x", "", 100)
objs, err = listAll(s, "x", "", 100, true)
if err != nil {
t.Fatalf("list failed: %s", err)
}
Expand All @@ -125,7 +125,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
t.Fatalf("testKeysEqual fail: %s", err)
}

objs, err = listAll(s, "xy", "", 100)
objs, err = listAll(s, "xy", "", 100, true)
if err != nil {
t.Fatalf("list failed: %s", err)
}
Expand Down Expand Up @@ -159,7 +159,7 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
}
_ = ss.Symlink("xyz/notExist/", "b")

objs, err = listAll(s, "", "", 100)
objs, err = listAll(s, "", "", 100, true)
if err != nil {
t.Fatalf("listall failed: %s", err)
}
Expand All @@ -173,5 +173,14 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
if objs[9].Size() != 10 {
t.Fatalf("size of target(file) should be 10")
}

// test don't follow symlink
if _, ok := s.(*hdfsclient); !ok {
objs, err = listAll(s, "", "", 100, false)
expectedKeys = []string{"", "a", "a-", "a0", "b", "b-", "b0", "bb/", "bb/b1", "x/", "x/x.txt", "xy.txt", "xyz/", "xyz/ol1/", "xyz/ol1/p.txt", "xyz/xyz.txt"}
if err = testKeysEqual(objs, expectedKeys); err != nil {
t.Fatalf("testKeysEqual fail: %s", err)
}
}
}
}
13 changes: 6 additions & 7 deletions pkg/object/gluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func (c *gluster) Delete(key string) error {

// readDirSorted reads the directory named by dirname and returns
// a sorted list of directory entries.
func (d *gluster) readDirSorted(dirname string) ([]*mEntry, error) {
func (d *gluster) readDirSorted(dirname string, followLink bool) ([]*mEntry, error) {
f, err := d.vol.Open(dirname)
if err != nil {
return nil, err
Expand All @@ -166,8 +166,7 @@ func (d *gluster) readDirSorted(dirname string) ([]*mEntry, error) {
}
if e.IsDir() {
mEntries = append(mEntries, &mEntry{nil, name + dirSuffix, e, false})
} else if !e.Mode().IsRegular() {
// follow symlink
} else if !e.Mode().IsRegular() && followLink {
fi, err := d.vol.Stat(filepath.Join(dirname, name))
if err != nil {
mEntries = append(mEntries, &mEntry{nil, name, e, true})
Expand All @@ -176,16 +175,16 @@ func (d *gluster) readDirSorted(dirname string) ([]*mEntry, error) {
if fi.IsDir() {
name += dirSuffix
}
mEntries = append(mEntries, &mEntry{nil, name, fi, true})
mEntries = append(mEntries, &mEntry{nil, name, fi, false})
} else {
mEntries = append(mEntries, &mEntry{nil, name, e, false})
mEntries = append(mEntries, &mEntry{nil, name, e, !e.Mode().IsRegular()})
}
}
sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })
return mEntries, err
}

func (d *gluster) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
func (d *gluster) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
if delimiter != "/" {
return nil, notSupported
}
Expand All @@ -206,7 +205,7 @@ func (d *gluster) List(prefix, marker, delimiter string, limit int64) ([]Object,
}
objs = append(objs, obj)
}
entries, err := d.readDirSorted(dir)
entries, err := d.readDirSorted(dir, followLink)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
Expand Down
4 changes: 2 additions & 2 deletions pkg/object/gs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func (g *gs) String() string {

func (g *gs) Create() error {
// check if the bucket is already exists
if objs, err := g.List("", "", "", 1); err == nil && len(objs) > 0 {
if objs, err := g.List("", "", "", 1, true); err == nil && len(objs) > 0 {
return nil
}

Expand Down Expand Up @@ -141,7 +141,7 @@ func (g *gs) Delete(key string) error {
return nil
}

func (g *gs) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
func (g *gs) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
if marker != "" && g.pageToken == "" {
// last page
return nil, nil
Expand Down
2 changes: 1 addition & 1 deletion pkg/object/hdfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ func (h *hdfsclient) Delete(key string) error {
return err
}

func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
if delimiter != "/" {
return nil, notSupported
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/object/ibmcos.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func (s *ibmcos) Delete(key string) error {
return err
}

func (s *ibmcos) List(prefix, marker, delimiter string, limit int64) ([]Object, error) {
func (s *ibmcos) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
param := s3.ListObjectsInput{
Bucket: &s.bucket,
Prefix: &prefix,
Expand Down Expand Up @@ -203,7 +203,7 @@ func (s *ibmcos) List(prefix, marker, delimiter string, limit int64) ([]Object,
return objs, nil
}

func (s *ibmcos) ListAll(prefix, marker string) (<-chan Object, error) {
func (s *ibmcos) ListAll(prefix, marker string, followLink bool) (<-chan Object, error) {
return nil, notSupported
}

Expand Down
4 changes: 2 additions & 2 deletions pkg/object/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,9 @@ type ObjectStorage interface {
// Head returns some information about the object or an error if not found.
Head(key string) (Object, error)
// List returns a list of objects.
List(prefix, marker, delimiter string, limit int64) ([]Object, error)
List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error)
// ListAll returns all the objects as an channel.
ListAll(prefix, marker string) (<-chan Object, error)
ListAll(prefix, marker string, followLink bool) (<-chan Object, error)

// CreateMultipartUpload starts to upload a large object part by part.
CreateMultipartUpload(key string) (*MultipartUpload, error)
Expand Down
Loading

0 comments on commit 66bdd05

Please sign in to comment.