enhance: support syncpeers by service and optimize the merge logic (#3637)

* enhance: support syncpeers by service and optimize the merge logic

Signed-off-by: cormick <cormick1080@gmail.com>

Author: CormickKneey

.github/workflows/lint.yml

@@ -23,8 +23,8 @@ jobs:
       - name: Golangci lint
         uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8
         with:
-          version: v1.54
-          args: --verbose
+          version: v1.62
+          args: --verbose --timeout=10m
 
       - name: Markdown lint
         uses: docker://avtodev/markdown-lint:v1@sha256:6aeedc2f49138ce7a1cd0adffc1b1c0321b841dc2102408967d9301c031949ee

.golangci.yml

@@ -24,11 +24,9 @@ linters:
   enable:
     - gci
     - gofmt
-    - golint
     - misspell
     - govet
     - goconst
-    - deadcode
     - gocyclo
     - staticcheck
     - errcheck

client-rs

@@ -338,6 +338,9 @@ type SyncPeersConfig struct {
 
 	// Timeout is the timeout for syncing peers information from the single scheduler.
 	Timeout time.Duration `yaml:"timeout" mapstructure:"timeout"`
+
+	// BatchSize is the batch size when operating gorm.
+	BatchSize int `yaml:"batchSize" mapstructure:"batchSize"`
 }
 
 type PreheatTLSClientConfig struct {
@@ -447,8 +450,9 @@ func New() *Config {
 				TLS:             PreheatTLSClientConfig{},
 			},
 			SyncPeers: SyncPeersConfig{
-				Interval: DefaultJobSyncPeersInterval,
-				Timeout:  DefaultJobSyncPeersTimeout,
+				Interval:  DefaultJobSyncPeersInterval,
+				Timeout:   DefaultJobSyncPeersTimeout,
+				BatchSize: DefaultJobSyncPeersBatchSize,
 			},
 		},
 		ObjectStorage: ObjectStorageConfig{

manager/config/config.go

@@ -107,6 +107,9 @@ const (
 
 	// DefaultClusterJobRateLimit is default rate limit(requests per second) for job Open API by cluster.
 	DefaultClusterJobRateLimit = 10
+
+	// DefaultJobSyncPeersBatchSize is the default batch size for syncing all peers information from the scheduler.
+	DefaultJobSyncPeersBatchSize = 500
 )
 
 const (

manager/config/constants.go

@@ -62,6 +62,20 @@ func (h *Handlers) CreateJob(ctx *gin.Context) {
 			return
 		}
 
+		ctx.JSON(http.StatusOK, job)
+	case job.SyncPeersJob:
+		var json types.CreateSyncPeersJobRequest
+		if err := ctx.ShouldBindBodyWith(&json, binding.JSON); err != nil {
+			ctx.JSON(http.StatusUnprocessableEntity, gin.H{"errors": err.Error()})
+			return
+		}
+
+		job, err := h.service.CreateSyncPeersJob(ctx.Request.Context(), json)
+		if err != nil {
+			ctx.Error(err) // nolint: errcheck
+			return
+		}
+
 		ctx.JSON(http.StatusOK, job)
 	case job.GetTaskJob:
 		var json types.CreateGetTaskJobRequest