diff --git a/internal/job/constants.go b/internal/job/constants.go index 0c580f1b83e..d9c475bcb22 100644 --- a/internal/job/constants.go +++ b/internal/job/constants.go @@ -43,8 +43,8 @@ const ( // Machinery server configuration. const ( DefaultResultsExpireIn = 86400 - DefaultRedisMaxIdle = 5 - DefaultRedisMaxActive = 20 + DefaultRedisMaxIdle = 30 + DefaultRedisMaxActive = 50 DefaultRedisIdleTimeout = 30 DefaultRedisReadTimeout = 60 DefaultRedisWriteTimeout = 60 diff --git a/manager/config/constants.go b/manager/config/constants.go index b1665c6365b..107b5efbc0e 100644 --- a/manager/config/constants.go +++ b/manager/config/constants.go @@ -58,7 +58,7 @@ const ( const ( // DefaultRedisPoolSize is default pool size for redis. - DefaultRedisPoolSize = 20 + DefaultRedisPoolSize = 40 // DefaultRedisPoolTimeout is default pool timeout for redis. DefaultRedisPoolTimeout = 10 * time.Second diff --git a/manager/database/database.go b/manager/database/database.go index e978c777473..471d8c6ca29 100644 --- a/manager/database/database.go +++ b/manager/database/database.go @@ -88,6 +88,8 @@ func New(cfg *config.Config) (*Database, error) { Password: cfg.Database.Redis.Password, SentinelUsername: cfg.Database.Redis.SentinelUsername, SentinelPassword: cfg.Database.Redis.SentinelPassword, + PoolSize: cfg.Database.Redis.PoolSize, + PoolTimeout: cfg.Database.Redis.PoolTimeout, }) if err != nil { return nil, err diff --git a/pkg/redis/redis.go b/pkg/redis/redis.go index 73ac37f3afd..4113ab7b0a3 100644 --- a/pkg/redis/redis.go +++ b/pkg/redis/redis.go @@ -74,6 +74,8 @@ func NewRedis(cfg *redis.UniversalOptions) (redis.UniversalClient, error) { Password: cfg.Password, SentinelUsername: cfg.SentinelUsername, SentinelPassword: cfg.SentinelPassword, + PoolSize: cfg.PoolSize, + PoolTimeout: cfg.PoolTimeout, }) if err := client.Ping(context.Background()).Err(); err != nil { diff --git a/scheduler/config/config.go b/scheduler/config/config.go index 7149aeb3c69..f47ef1a2e71 100644 --- a/scheduler/config/config.go +++ b/scheduler/config/config.go @@ -283,6 +283,12 @@ type RedisConfig struct { // SentinelPassword is sentinel server password. SentinelPassword string `yaml:"sentinelPassword" mapstructure:"sentinelPassword"` + // PoolSize is the maximum number of idle connections in the pool. + PoolSize int `yaml:"poolSize" mapstructure:"poolSize"` + + // PoolTimeout is the maximum amount of time a connection may be idle before being closed. + PoolTimeout time.Duration `yaml:"poolTimeout" mapstructure:"poolTimeout"` + // BrokerDB is broker database name. BrokerDB int `yaml:"brokerDB" mapstructure:"brokerDB"` @@ -334,8 +340,10 @@ func New() *Config { }, Database: DatabaseConfig{ Redis: RedisConfig{ - BrokerDB: DefaultRedisBrokerDB, - BackendDB: DefaultRedisBackendDB, + BrokerDB: DefaultRedisBrokerDB, + BackendDB: DefaultRedisBackendDB, + PoolSize: DefaultRedisPoolSize, + PoolTimeout: DefaultRedisPoolTimeout, }, }, DynConfig: DynConfig{ diff --git a/scheduler/config/config_test.go b/scheduler/config/config_test.go index 58ae00ec060..a48acfd9504 100644 --- a/scheduler/config/config_test.go +++ b/scheduler/config/config_test.go @@ -95,13 +95,15 @@ func TestConfig_Load(t *testing.T) { }, Database: DatabaseConfig{ Redis: RedisConfig{ - Host: "127.0.0.1", - Password: "foo", - Addrs: []string{"foo", "bar"}, - MasterName: "baz", - Port: 6379, - BrokerDB: DefaultRedisBrokerDB, - BackendDB: DefaultRedisBackendDB, + Host: "127.0.0.1", + Password: "foo", + Addrs: []string{"foo", "bar"}, + MasterName: "baz", + Port: 6379, + BrokerDB: DefaultRedisBrokerDB, + BackendDB: DefaultRedisBackendDB, + PoolSize: 10, + PoolTimeout: 1 * time.Second, }, }, DynConfig: DynConfig{ diff --git a/scheduler/config/constants.go b/scheduler/config/constants.go index 4543311d710..79ae2751871 100644 --- a/scheduler/config/constants.go +++ b/scheduler/config/constants.go @@ -51,6 +51,12 @@ const ( // DefaultRedisBackendDB is default db for redis backend. DefaultRedisBackendDB = 2 + + // DefaultRedisPoolSize is default pool size for redis. + DefaultRedisPoolSize = 20 + + // DefaultRedisPoolTimeout is default pool timeout for redis. + DefaultRedisPoolTimeout = 10 * time.Second ) const ( diff --git a/scheduler/config/testdata/scheduler.yaml b/scheduler/config/testdata/scheduler.yaml index 1ae24a051f3..05f1f715def 100644 --- a/scheduler/config/testdata/scheduler.yaml +++ b/scheduler/config/testdata/scheduler.yaml @@ -39,6 +39,8 @@ database: password: foo brokerDB: 1 backendDB: 2 + poolSize: 10 + poolTimeout: 1s dynConfig: refreshInterval: 10s diff --git a/scheduler/scheduler.go b/scheduler/scheduler.go index ec7c0124a71..9d347e2bfda 100644 --- a/scheduler/scheduler.go +++ b/scheduler/scheduler.go @@ -122,6 +122,8 @@ func New(ctx context.Context, cfg *config.Config, d dfpath.Dfpath) (*Server, err Password: cfg.Database.Redis.Password, SentinelUsername: cfg.Database.Redis.SentinelUsername, SentinelPassword: cfg.Database.Redis.SentinelPassword, + PoolSize: cfg.Database.Redis.PoolSize, + PoolTimeout: cfg.Database.Redis.PoolTimeout, }) if err != nil { return nil, err