From 9a926a498bea65973f7b52d50efd50d43cfc1d90 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Tue, 25 Jul 2023 12:56:12 +0800 Subject: [PATCH] sync: read large file from sftp in parallel --- pkg/sync/sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index a6697eef42f5..d0527ebf9519 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -322,7 +322,7 @@ func checkSum(src, dst object.ObjectStorage, key string, size int64) (bool, erro return equal, err } -var streamRead = map[string]struct{}{"file": {}, "hdfs": {}, "sftp": {}, "gluster": {}} +var fastStreamRead = map[string]struct{}{"file": {}, "hdfs": {}, "jfs": {}, "gluster": {}} var streamWrite = map[string]struct{}{"file": {}, "hdfs": {}, "sftp": {}, "gs": {}, "wasb": {}, "ceph": {}, "swift": {}, "webdav": {}, "upyun": {}, "jfs": {}, "gluster": {}} var readInMem = map[string]struct{}{"mem": {}, "etcd": {}, "redis": {}, "tikv": {}, "mysql": {}, "postgres": {}, "sqlite3": {}} @@ -332,7 +332,7 @@ func inMap(obj object.ObjectStorage, m map[string]struct{}) bool { } func doCopySingle(src, dst object.ObjectStorage, key string, size int64) error { - if size > maxBlock && !inMap(dst, readInMem) && !inMap(src, streamRead) { + if size > maxBlock && !inMap(dst, readInMem) && !inMap(src, fastStreamRead) { var err error var in io.Reader downer := newParallelDownloader(src, key, size, 10<<20, concurrent)