feat: add cacher for gorm to reduce the load of db

go.mod

@@ -22,13 +22,15 @@ require (
 	github.com/docker/docker v27.3.1+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/docker/go-units v0.4.0
+	github.com/elastic/go-freelru v0.16.0
 	github.com/gaius-qi/ping v1.0.0
 	github.com/gammazero/deque v1.0.0
 	github.com/gin-contrib/gzip v1.0.1
 	github.com/gin-contrib/static v1.1.2
 	github.com/gin-contrib/zap v1.1.3
 	github.com/gin-gonic/gin v1.10.0
 	github.com/go-echarts/statsview v0.3.4
+	github.com/go-gorm/caches/v4 v4.0.5
 	github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a
 	github.com/go-playground/validator/v10 v10.23.0
 	github.com/go-redis/cache/v9 v9.0.0
@@ -94,7 +96,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/driver/mysql v1.4.7
 	gorm.io/driver/postgres v1.4.8
-	gorm.io/gorm v1.24.6
+	gorm.io/gorm v1.25.0
 	gorm.io/plugin/soft_delete v1.2.1
 	k8s.io/component-base v0.31.2
 	moul.io/zapgorm2 v1.3.0

go.sum

@@ -468,6 +468,8 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elastic/go-freelru v0.16.0 h1:gG2HJ1WXN2tNl5/p40JS/l59HjvjRhjyAa+oFTRArYs=
+github.com/elastic/go-freelru v0.16.0/go.mod h1:bSdWT4M0lW79K8QbX6XY2heQYSCqD7THoYf82pT/H3I=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -531,6 +533,8 @@ github.com/go-echarts/statsview v0.3.4/go.mod h1:AehKjL9cTFMeIo5QdV8sQO43vFmfY65
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gorm/caches/v4 v4.0.5 h1:Sdj9vxbEM0sCmv5+s5o6GzoVMuraWF0bjJJvUU+7c1U=
+github.com/go-gorm/caches/v4 v4.0.5/go.mod h1:Ms8LnWVoW4GkTofpDzFH8OfDGNTjLxQDyxBmRN67Ujw=
 github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=
 github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw=
 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
@@ -2272,8 +2276,8 @@ gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
 gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
 gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
 gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA=
-gorm.io/gorm v1.24.6 h1:wy98aq9oFEetsc4CAbKD2SoBCdMzsbSIvSUUFJuHi5s=
-gorm.io/gorm v1.24.6/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
+gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU=
+gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
 gorm.io/plugin/dbresolver v1.1.0/go.mod h1:tpImigFAEejCALOttyhWqsy4vfa2Uh/vAUVnL5IRF7Y=
 gorm.io/plugin/dbresolver v1.3.0 h1:uFDX3bIuH9Lhj5LY2oyqR/bU6pqWuDgas35NAPF4X3M=
 gorm.io/plugin/dbresolver v1.3.0/go.mod h1:Pr7p5+JFlgDaiM6sOrli5olekJD16YRunMyA2S7ZfKk=

manager/config/constants.go

@@ -75,7 +75,7 @@ const (
 	DefaultLFUCacheTTL = 3 * time.Minute
 
 	// DefaultLFUCacheSize is default size for lfu cache.
-	DefaultLFUCacheSize = 50 * 1000
+	DefaultLFUCacheSize = 30 * 1000
 )
 
 const (
@@ -88,10 +88,10 @@ const (
 
 const (
 	// DefaultJobGCInterval is the default interval for gc job.
-	DefaultJobGCInterval = 6 * time.Hour
+	DefaultJobGCInterval = 3 * time.Hour
 
 	// DefaultJobGCTTL is the default ttl for job.
-	DefaultJobGCTTL = 12 * time.Hour
+	DefaultJobGCTTL = 6 * time.Hour
 
 	// DefaultJobGCBatchSize is the default batch size for operating on the database in gc job.
 	DefaultJobGCBatchSize = 5000

manager/database/cacher.go

@@ -0,0 +1,96 @@
+/*
+ *     Copyright 2024 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package database
+
+import (
+	"context"
+	"time"
+
+	xxhash "github.com/cespare/xxhash/v2"
+	lru "github.com/elastic/go-freelru"
+	caches "github.com/go-gorm/caches/v4"
+)
+
+const (
+	// defaultCacheSize is the default size of the cache.
+	defaultCacheSize = 1024
+
+	// defaultTTL is the default TTL of the cache.
+	defaultTTL = 30 * time.Second
+)
+
+// cacher is a cache implementation using LRU for gorm.
+type cacher struct {
+	// store is the LRU cache.
+	store *lru.LRU[string, any]
+}
+
+// hashStringXXHASH returns the hash of the string s.
+func hashStringXXHASH(s string) uint32 {
+	return uint32(xxhash.Sum64String(s))
+}
+
+// newCacher creates a new cacher.
+func newCacher() (caches.Cacher, error) {
+	store, err := lru.New[string, any](defaultCacheSize, hashStringXXHASH)
+	if err != nil {
+		return nil, err
+	}
+
+	store.SetLifetime(defaultTTL)
+	return &cacher{store: store}, nil
+}
+
+// Get impl should check if a specific key exists in the cache and return its value
+// look at Query.Marshal.
+func (c *cacher) Get(ctx context.Context, key string, q *caches.Query[any]) (*caches.Query[any], error) {
+	val, ok := c.store.Get(key)
+	if !ok {
+		return nil, nil
+	}
+
+	if err := q.Unmarshal(val.([]byte)); err != nil {
+		return nil, err
+	}
+
+	return q, nil
+}
+
+// Store impl should store a cached representation of the val param
+// look at Query.Unmarshal.
+func (c *cacher) Store(ctx context.Context, key string, val *caches.Query[any]) error {
+	res, err := val.Marshal()
+	if err != nil {
+		return err
+	}
+
+	c.store.Add(key, res)
+	return nil
+}
+
+// Invalidate impl should invalidate all cached values. It will be called when
+// INSERT / UPDATE / DELETE queries are sent to the DB.
+func (c *cacher) Invalidate(ctx context.Context) error {
+	var err error
+	c.store, err = lru.New[string, any](defaultCacheSize, hashStringXXHASH)
+	if err != nil {
+		return err
+	}
+
+	c.store.SetLifetime(defaultTTL)
+	return nil
+}

manager/database/database.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 
+	caches "github.com/go-gorm/caches/v4"
 	"github.com/redis/go-redis/v9"
 	"gorm.io/gorm"
 
@@ -63,6 +64,21 @@ func New(cfg *config.Config) (*Database, error) {
 		return nil, fmt.Errorf("invalid database type %s", cfg.Database.Type)
 	}
 
+	// Add cache for reducing database load.
+	cacher, err := newCacher()
+	if err != nil {
+		logger.Errorf("new cacher: %s", err.Error())
+		return nil, err
+	}
+
+	if err := db.Use(&caches.Caches{Conf: &caches.Config{
+		Easer:  true,
+		Cacher: cacher,
+	}}); err != nil {
+		logger.Errorf("use cache: %s", err.Error())
+		return nil, err
+	}
+
 	rdb, err := pkgredis.NewRedis(&redis.UniversalOptions{
 		Addrs:            cfg.Database.Redis.Addrs,
 		MasterName:       cfg.Database.Redis.MasterName,