diff --git a/.github/scripts/setup-hdfs.sh b/.github/scripts/setup-hdfs.sh index 582b51513a77..3c3ea96decf4 100755 --- a/.github/scripts/setup-hdfs.sh +++ b/.github/scripts/setup-hdfs.sh @@ -90,5 +90,6 @@ echo "hello world" > /tmp/testfile cd ~/app/hadoop-${HADOOP_VERSION}/bin ./hdfs dfs -put /tmp/testfile / ./hdfs dfs -rm /testfile +./hdfs dfs -chmod 777 / -echo "hdfs started successfully" \ No newline at end of file +echo "hdfs started successfully" diff --git a/.github/workflows/dockerfile-sftp b/.github/workflows/dockerfile-sftp index 5e662d87d527..77bcf244ce0a 100644 --- a/.github/workflows/dockerfile-sftp +++ b/.github/workflows/dockerfile-sftp @@ -1,5 +1,4 @@ -FROM debian:stretch-slim -RUN sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list +FROM debian:stable-slim RUN apt-get clean RUN apt-get update @@ -9,5 +8,7 @@ RUN mkdir /run/sshd RUN sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config RUN echo "root:password"|chpasswd +RUN useradd -m testUser1 +RUN echo "testUser1:password"|chpasswd EXPOSE 22 CMD ["/usr/sbin/sshd", "-D"] diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 4131422c559a..debf98b5fea9 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -39,8 +39,8 @@ jobs: GLUSTER_VOLUME: jfstest/gv0 DISPLAY_PROGRESSBAR: false HDFS_ADDR: localhost:8020 - SFTP_HOST: localhost:2222:/upload/ - SFTP_USER: root + SFTP_HOST: localhost:2222:/home/testUser1/upload/ + SFTP_USER: testUser1 SFTP_PASS: password WEBDAV_TEST_BUCKET: 127.0.0.1:9007 TIKV_ADDR: 127.0.0.1 diff --git a/pkg/object/file.go b/pkg/object/file.go index fea247ef7733..1df31b4c4100 100644 --- a/pkg/object/file.go +++ b/pkg/object/file.go @@ -220,6 +220,10 @@ func walk(path string, info os.FileInfo, isSymlink bool, walkFn WalkFunc) error if err == nil { err = walk(p, in, e.isSymlink, walkFn) } + if os.IsPermission(err) { + logger.Warnf("skip %s: %s", path, err) + break + } if err != nil && err != filepath.SkipDir && !os.IsNotExist(err) { return err } @@ -338,6 +342,10 @@ func (d *filestore) ListAll(prefix, marker string) (<-chan Object, error) { } if err != nil { + if os.IsPermission(err) { + logger.Warnf("skip %s: %s", path, err) + return nil + } if os.IsNotExist(err) { logger.Warnf("skip not exist file or directory: %s", path) return nil diff --git a/pkg/object/filesystem_test.go b/pkg/object/filesystem_test.go index 8c88e688eae4..de37e78f4522 100644 --- a/pkg/object/filesystem_test.go +++ b/pkg/object/filesystem_test.go @@ -63,7 +63,7 @@ func TestHDFS2(t *testing.T) { if os.Getenv("HDFS_ADDR") == "" { t.Skip() } - dfs, _ := newHDFS(os.Getenv("HDFS_ADDR"), "", "", "") + dfs, _ := newHDFS(os.Getenv("HDFS_ADDR"), "testUser1", "", "") testFileSystem(t, dfs) } @@ -124,6 +124,28 @@ func testFileSystem(t *testing.T, s ObjectStorage) { t.Fatalf("testKeysEqual fail: %s", err) } + if ss, ok := s.(FileSystem); ok { + for _, mode := range []uint32{0022, 0122, 0422} { + t.Logf("test mode %o", os.FileMode(mode)) + err := ss.Chmod("x/", os.FileMode(mode)) + if err != nil { + t.Fatalf("chmod %ofailed: %s", mode, err) + } + objs, err = listAll(s, "x", "", 100) + if err != nil { + t.Fatalf("list failed: %s mode %o", err, mode) + } + expectedKeys = []string{"x/", "xy.txt", "xyz/", "xyz/xyz.txt"} + if err = testKeysEqual(objs, expectedKeys); err != nil { + t.Fatalf("testKeysEqual fail: %s mode %o", err, mode) + } + err = ss.Chmod("x/", os.FileMode(0777)) + if err != nil { + t.Fatalf("chmod %o failed: %s", mode, err) + } + } + } + objs, err = listAll(s, "xy", "", 100) if err != nil { t.Fatalf("list failed: %s", err) diff --git a/pkg/object/hdfs.go b/pkg/object/hdfs.go index 2e1af728c00b..faeccec433b6 100644 --- a/pkg/object/hdfs.go +++ b/pkg/object/hdfs.go @@ -233,6 +233,10 @@ func (h *hdfsclient) ListAll(prefix, marker string) (<-chan Object, error) { go func() { _ = h.walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { + if os.IsPermission(err) { + logger.Warnf("skip %s: %s", path, err) + return nil + } if err == io.EOF { err = nil // ignore } else { diff --git a/pkg/object/webdav.go b/pkg/object/webdav.go index b5a43ee51252..ca8968873300 100644 --- a/pkg/object/webdav.go +++ b/pkg/object/webdav.go @@ -240,6 +240,10 @@ func (w *webdav) ListAll(prefix, marker string) (<-chan Object, error) { defer close(listed) _ = w.Walk(root, func(path string, info fs.FileInfo, err error) error { if err != nil { + if gowebdav.IsErrCode(err, http.StatusForbidden) { + logger.Warnf("skip %s: %s", path, err) + return nil + } if gowebdav.IsErrNotFound(err) { logger.Warnf("skip not exist file or directory: %s", path) return nil