enhance: support syncpeers by service and optimize the merge logic (#…

…3637) * enhance: support syncpeers by service and optimize the merge logic Signed-off-by: cormick <[email protected]>
dragonflyoss · Nov 29, 2024 · d0e41b5 · d0e41b5
1 parent c220a60
commit d0e41b5
Show file tree

Hide file tree

Showing 16 changed files with 543 additions and 112 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -23,8 +23,8 @@ jobs:
       - name: Golangci lint
         uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8
         with:
-          version: v1.54
-          args: --verbose
+          version: v1.62
+          args: --verbose --timeout=10m
 
       - name: Markdown lint
         uses: docker://avtodev/markdown-lint:v1@sha256:6aeedc2f49138ce7a1cd0adffc1b1c0321b841dc2102408967d9301c031949ee

diff --git a/.golangci.yml b/.golangci.yml
@@ -24,11 +24,9 @@ linters:
   enable:
     - gci
     - gofmt
-    - golint
     - misspell
     - govet
     - goconst
-    - deadcode
     - gocyclo
     - staticcheck
     - errcheck

diff --git a/client-rs b/client-rs
diff --git a/manager/config/config.go b/manager/config/config.go
@@ -338,6 +338,9 @@ type SyncPeersConfig struct {
 
 	// Timeout is the timeout for syncing peers information from the single scheduler.
 	Timeout time.Duration `yaml:"timeout" mapstructure:"timeout"`
+
+	// BatchSize is the batch size when operating gorm.
+	BatchSize int `yaml:"batchSize" mapstructure:"batchSize"`
 }
 
 type PreheatTLSClientConfig struct {
@@ -447,8 +450,9 @@ func New() *Config {
 				TLS:             PreheatTLSClientConfig{},
 			},
 			SyncPeers: SyncPeersConfig{
-				Interval: DefaultJobSyncPeersInterval,
-				Timeout:  DefaultJobSyncPeersTimeout,
+				Interval:  DefaultJobSyncPeersInterval,
+				Timeout:   DefaultJobSyncPeersTimeout,
+				BatchSize: DefaultJobSyncPeersBatchSize,
 			},
 		},
 		ObjectStorage: ObjectStorageConfig{

diff --git a/manager/config/constants.go b/manager/config/constants.go
@@ -107,6 +107,9 @@ const (
 
 	// DefaultClusterJobRateLimit is default rate limit(requests per second) for job Open API by cluster.
 	DefaultClusterJobRateLimit = 10
+
+	// DefaultJobSyncPeersBatchSize is the default batch size for syncing all peers information from the scheduler.
+	DefaultJobSyncPeersBatchSize = 500
 )
 
 const (

diff --git a/manager/handlers/job.go b/manager/handlers/job.go
@@ -62,6 +62,20 @@ func (h *Handlers) CreateJob(ctx *gin.Context) {
 			return
 		}
 
+		ctx.JSON(http.StatusOK, job)
+	case job.SyncPeersJob:
+		var json types.CreateSyncPeersJobRequest
+		if err := ctx.ShouldBindBodyWith(&json, binding.JSON); err != nil {
+			ctx.JSON(http.StatusUnprocessableEntity, gin.H{"errors": err.Error()})
+			return
+		}
+
+		job, err := h.service.CreateSyncPeersJob(ctx.Request.Context(), json)
+		if err != nil {
+			ctx.Error(err) // nolint: errcheck
+			return
+		}
+
 		ctx.JSON(http.StatusOK, job)
 	case job.GetTaskJob:
 		var json types.CreateGetTaskJobRequest

diff --git a/manager/job/mocks/sync_peers_mock.go b/manager/job/mocks/sync_peers_mock.go
+1 −1		.github/workflows/ci.yml
+54 −91		Cargo.lock
+13 −14		Cargo.toml
+1 −1		dragonfly-client-backend/Cargo.toml
+0 −274		dragonfly-client-backend/src/hdfs.rs
+40 −73		dragonfly-client-backend/src/http.rs
+19 −57		dragonfly-client-backend/src/lib.rs
+50 −59		dragonfly-client-backend/src/object_storage.rs
+2 −2		dragonfly-client-config/src/dfdaemon.rs
+0 −71		dragonfly-client-init/src/container_runtime/containerd.rs
+0 −51		dragonfly-client-init/src/container_runtime/crio.rs
+24 −19		dragonfly-client-storage/src/content.rs
+31 −29		dragonfly-client-storage/src/lib.rs
+83 −80		dragonfly-client-storage/src/metadata.rs
+20 −18		dragonfly-client-storage/src/storage_engine/rocksdb.rs
+105 −29		dragonfly-client-util/src/http/mod.rs
+2 −2		dragonfly-client/Cargo.toml
+6 −21		dragonfly-client/src/announcer/mod.rs
+7 −28		dragonfly-client/src/bin/dfdaemon/main.rs
+16 −42		dragonfly-client/src/bin/dfget/main.rs
+26 −32		dragonfly-client/src/grpc/dfdaemon_download.rs
+31 −33		dragonfly-client/src/grpc/dfdaemon_upload.rs
+1 −1		dragonfly-client/src/grpc/mod.rs
+11 −11		dragonfly-client/src/grpc/scheduler.rs
+107 −207		dragonfly-client/src/metrics/mod.rs
+31 −13		dragonfly-client/src/proxy/header.rs
+164 −185		dragonfly-client/src/proxy/mod.rs
+21 −14		dragonfly-client/src/resource/persistent_cache_task.rs
+31 −46		dragonfly-client/src/resource/piece.rs
+13 −10		dragonfly-client/src/resource/piece_collector.rs
+75 −68		dragonfly-client/src/resource/task.rs
+2 −3		dragonfly-client/src/tracing/mod.rs