Skip to content

Commit c4f5e8b

Browse files
oliver006oliver006
authored
Add flag to skip checkKeys if role == master (#985)
* add flag to skip checkKeys if role == master (can help with reducing load on master nodes) - see #931
1 parent b54c350 commit c4f5e8b

11 files changed

+166
-124
lines changed

‎exporter/exporter.go

+14-15
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ type Options struct {
8787
BuildInfo BuildInfo
8888
BasicAuthUsername string
8989
BasicAuthPassword string
90+
SkipCheckKeysForRoleMaster bool
9091
}
9192

9293
// NewRedisExporter returns a new exporter of Redis metrics.
@@ -197,7 +198,7 @@ func NewRedisExporter(uri string, opts Options) (*Exporter, error) {
197198
"expired_stale_perc": "expired_stale_percentage",
198199

199200
// https://github.com/antirez/redis/blob/17bf0b25c1171486e3a1b089f3181fff2bc0d4f0/src/evict.c#L349-L352
200-
// ... the sum of AOF and slaves buffer ....
201+
// ... the sum of AOF and slaves buffer ...
201202
"mem_not_counted_for_evict": "mem_not_counted_for_eviction_bytes",
202203
"mem_total_replication_buffers": "mem_total_replication_buffers_bytes", // Added in Redis 7.0
203204
"mem_overhead_db_hashtable_rehashing": "mem_overhead_db_hashtable_rehashing_bytes", // Added in Redis 7.4
@@ -720,38 +721,36 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error {
720721
}
721722
} else if dbCount == 0 {
722723
// in non-cluster mode, if dbCount is zero then "CONFIG" failed to retrieve a valid
723-
// number of databases and we use the Redis config default which is 16
724+
// number of databases, and we use the Redis config default which is 16
724725

725726
dbCount = 16
726727
}
727728

728729
log.Debugf("dbCount: %d", dbCount)
729730

730-
e.extractInfoMetrics(ch, infoAll, dbCount)
731+
role := e.extractInfoMetrics(ch, infoAll, dbCount)
731732

732733
if !e.options.ExcludeLatencyHistogramMetrics {
733734
e.extractLatencyMetrics(ch, infoAll, c)
734735
}
735736

736-
if e.options.IsCluster {
737-
clusterClient, err := e.connectToRedisCluster()
738-
if err != nil {
739-
log.Errorf("Couldn't connect to redis cluster")
740-
return err
737+
// skip these metrics for master if SkipCheckKeysForRoleMaster is set
738+
// (can help with reducing workload on the master node)
739+
log.Infof("checkKeys metric collection for role: %s flag: %#v", role, e.options.SkipCheckKeysForRoleMaster)
740+
if role == InstanceRoleSlave || !e.options.SkipCheckKeysForRoleMaster {
741+
if err := e.extractCheckKeyMetrics(ch, c); err != nil {
742+
log.Errorf("extractCheckKeyMetrics() err: %s", err)
741743
}
742-
defer clusterClient.Close()
743744

744-
e.extractCheckKeyMetrics(ch, clusterClient)
745+
e.extractCountKeysMetrics(ch, c)
746+
747+
e.extractStreamMetrics(ch, c)
745748
} else {
746-
e.extractCheckKeyMetrics(ch, c)
749+
log.Infof("skipping checkKeys metrics, role: %s flag: %#v", role, e.options.SkipCheckKeysForRoleMaster)
747750
}
748751

749752
e.extractSlowLogMetrics(ch, c)
750753

751-
e.extractStreamMetrics(ch, c)
752-
753-
e.extractCountKeysMetrics(ch, c)
754-
755754
e.extractKeyGroupMetrics(ch, c, dbCount)
756755

757756
if strings.Contains(infoAll, "# Sentinel") {

‎exporter/exporter_test.go

+54-40
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,12 @@ import (
2424
log "github.com/sirupsen/logrus"
2525
)
2626

27-
var (
28-
dbNumStr = "11"
29-
altDBNumStr = "12"
30-
anotherAltDbNumStr = "14"
31-
27+
const (
28+
dbNumStr = "11"
29+
altDBNumStr = "12"
3230
invalidDBNumStr = "16"
33-
dbNumStrFull = fmt.Sprintf("db%s", dbNumStr)
31+
32+
anotherAltDbNumStr = "14"
3433
)
3534

3635
const (
@@ -44,11 +43,6 @@ const (
4443
)
4544

4645
var (
47-
testKeySingleString string
48-
testKeys []string
49-
testKeysExpiring []string
50-
testKeysList []string
51-
5246
AllTestKeys = []string{
5347
TestKeysSetName, TestKeysZSetName,
5448
TestKeysStreamName,
@@ -57,6 +51,21 @@ var (
5751
}
5852
)
5953

54+
var (
55+
testKeys []string
56+
testKeysExpiring []string
57+
testKeysList []string
58+
59+
dbNumStrFull = fmt.Sprintf("db%s", dbNumStr)
60+
)
61+
62+
var (
63+
TestKeyNameSingleString = "" // initialized with a timestamp at runtime
64+
TestKeyNameSet = "test-set"
65+
TestKeyNameStream = "test-stream"
66+
TestKeyNameHll = "test-hll"
67+
)
68+
6069
func getTestExporter() *Exporter {
6170
return getTestExporterWithOptions(Options{Namespace: "test", Registry: prometheus.NewRegistry()})
6271
}
@@ -111,30 +120,30 @@ func setupKeys(t *testing.T, c redis.Conn, dbNum string) error {
111120
}
112121
}
113122

114-
if _, err := doRedisCmd(c, "PFADD", TestKeysHllName, "val1"); err != nil {
123+
if _, err := c.Do("PFADD", TestKeyNameHll, "val1"); err != nil {
115124
t.Errorf("PFADD err: %s", err)
116125
return err
117126
}
118-
if _, err := doRedisCmd(c, "PFADD", TestKeysHllName, "val22"); err != nil {
127+
if _, err := c.Do("PFADD", TestKeyNameHll, "val22"); err != nil {
119128
t.Errorf("PFADD err: %s", err)
120129
return err
121130
}
122-
if _, err := doRedisCmd(c, "PFADD", TestKeysHllName, "val333"); err != nil {
131+
if _, err := c.Do("PFADD", TestKeyNameHll, "val333"); err != nil {
123132
t.Errorf("PFADD err: %s", err)
124133
return err
125134
}
126135

127-
if _, err := doRedisCmd(c, "SADD", TestKeysSetName, "test-val-1"); err != nil {
136+
if _, err := c.Do("SADD", TestKeyNameSet, "test-val-1"); err != nil {
128137
t.Errorf("SADD err: %s", err)
129138
return err
130139
}
131-
if _, err := doRedisCmd(c, "SADD", TestKeysSetName, "test-val-2"); err != nil {
140+
if _, err := c.Do("SADD", TestKeyNameSet, "test-val-2"); err != nil {
132141
t.Errorf("SADD err: %s", err)
133142
return err
134143
}
135144

136-
if _, err := doRedisCmd(c, "ZADD", TestKeysZSetName, "12", "test-zzzval-1"); err != nil {
137-
t.Errorf("ZADD err: %s", err)
145+
if _, err := c.Do("SET", TestKeyNameSingleString, "this-is-a-string"); err != nil {
146+
t.Errorf("PFADD err: %s", err)
138147
return err
139148
}
140149
if _, err := doRedisCmd(c, "ZADD", TestKeysZSetName, "23", "test-zzzval-2"); err != nil {
@@ -146,8 +155,8 @@ func setupKeys(t *testing.T, c redis.Conn, dbNum string) error {
146155
return err
147156
}
148157

149-
if _, err := doRedisCmd(c, "SET", testKeySingleString, "this-is-a-string"); err != nil {
150-
t.Errorf("SET %s err: %s", testKeySingleString, err)
158+
if _, err := doRedisCmd(c, "SET", TestKeyNameSingleString, "this-is-a-string"); err != nil {
159+
t.Errorf("SET %s err: %s", TestKeyNameSingleString, err)
151160
return err
152161
}
153162

@@ -173,17 +182,15 @@ func setupKeys(t *testing.T, c redis.Conn, dbNum string) error {
173182
}
174183

175184
// Create test streams
176-
doRedisCmd(c, "XGROUP", "CREATE", TestKeysStreamName, TestKeyGroup1, "$", "MKSTREAM")
177-
doRedisCmd(c, "XGROUP", "CREATE", TestKeysStreamName, TestKeyGroup2, "$", "MKSTREAM")
178-
doRedisCmd(c, "XADD", TestKeysStreamName, TestStreamTimestamps[0], "field_1", "str_1")
179-
doRedisCmd(c, "XADD", TestKeysStreamName, TestStreamTimestamps[1], "field_2", "str_2")
185+
c.Do("XGROUP", "CREATE", TestKeyNameStream, "test_group_1", "$", "MKSTREAM")
186+
c.Do("XGROUP", "CREATE", TestKeyNameStream, "test_group_2", "$", "MKSTREAM")
187+
c.Do("XADD", TestKeyNameStream, TestStreamTimestamps[0], "field_1", "str_1")
188+
c.Do("XADD", TestKeyNameStream, TestStreamTimestamps[1], "field_2", "str_2")
180189

181190
// Process messages to assign Consumers to their groups
182-
doRedisCmd(c, "XREADGROUP", "GROUP", TestKeyGroup1, "test_consumer_1", "COUNT", "1", "STREAMS", TestKeysStreamName, ">")
183-
doRedisCmd(c, "XREADGROUP", "GROUP", TestKeyGroup1, "test_consumer_2", "COUNT", "1", "STREAMS", TestKeysStreamName, ">")
184-
doRedisCmd(c, "XREADGROUP", "GROUP", TestKeyGroup2, "test_consumer_1", "COUNT", "1", "STREAMS", TestKeysStreamName, "0")
185-
186-
t.Logf("setupKeys %s - DONE", dbNum)
191+
c.Do("XREADGROUP", "GROUP", "test_group_1", "test_consumer_1", "COUNT", "1", "STREAMS", TestKeyNameStream, ">")
192+
c.Do("XREADGROUP", "GROUP", "test_group_1", "test_consumer_2", "COUNT", "1", "STREAMS", TestKeyNameStream, ">")
193+
c.Do("XREADGROUP", "GROUP", "test_group_2", "test_consumer_1", "COUNT", "1", "STREAMS", TestKeyNameStream, "0")
187194

188195
time.Sleep(time.Millisecond * 100)
189196
return nil
@@ -198,6 +205,19 @@ func deleteKeys(c redis.Conn, dbNum string) {
198205
for _, key := range AllTestKeys {
199206
doRedisCmd(c, "DEL", key)
200207
}
208+
209+
for _, key := range testKeysExpiring {
210+
c.Do("DEL", key)
211+
}
212+
213+
for _, key := range testKeysList {
214+
c.Do("DEL", key)
215+
}
216+
217+
c.Do("DEL", TestKeyNameHll)
218+
c.Do("DEL", TestKeyNameSet)
219+
c.Do("DEL", TestKeyNameStream)
220+
c.Do("DEL", TestKeyNameSingleString)
201221
}
202222

203223
func setupTestKeys(t *testing.T, uri string) {
@@ -273,10 +293,11 @@ func deleteTestKeys(t *testing.T, addr string) error {
273293
return nil
274294
}
275295

276-
func deleteTestKeysCluster(addr string) error {
296+
func deleteTestKeysCluster(t *testing.T, addr string) error {
277297
e, _ := NewRedisExporter(addr, Options{})
278298
c, err := e.connectToRedisCluster()
279299
if err != nil {
300+
t.Errorf("couldn't setup redis CLUSTER, err: %s ", err)
280301
return err
281302
}
282303

@@ -433,15 +454,9 @@ func TestKeysReset(t *testing.T) {
433454
setupTestKeys(t, os.Getenv("TEST_REDIS_URI"))
434455
defer deleteTestKeys(t, os.Getenv("TEST_REDIS_URI"))
435456

436-
chM := make(chan prometheus.Metric, 10000)
437-
go func() {
438-
e.Collect(chM)
439-
close(chM)
440-
}()
441-
442457
body := downloadURL(t, ts.URL+"/metrics")
443458
if !strings.Contains(body, testKeys[0]) {
444-
t.Errorf("Did not found key %q\n%s", testKeys[0], body)
459+
t.Errorf("Did not find key %q\n%s", testKeys[0], body)
445460
}
446461

447462
deleteTestKeys(t, os.Getenv("TEST_REDIS_URI"))
@@ -538,15 +553,14 @@ func init() {
538553
testKeys = append(testKeys, fmt.Sprintf("key_%s_%d", n, testTimestamp))
539554
}
540555

541-
testKeySingleString = fmt.Sprintf("key_string_%d", testTimestamp)
542-
AllTestKeys = append(AllTestKeys, testKeySingleString)
543-
556+
TestKeyNameSingleString = fmt.Sprintf("key_string_%d", testTimestamp)
544557
testKeysList = append(testKeysList, "test_beatles_list")
545558

546559
for _, n := range []string{"A.J.", "Howie", "Nick", "Kevin", "Brian"} {
547560
testKeysExpiring = append(testKeysExpiring, fmt.Sprintf("key_exp_%s_%d", n, testTimestamp))
548561
}
549562

563+
AllTestKeys = append(AllTestKeys, TestKeyNameSingleString)
550564
AllTestKeys = append(AllTestKeys, testKeys...)
551565
AllTestKeys = append(AllTestKeys, testKeysList...)
552566
AllTestKeys = append(AllTestKeys, testKeysExpiring...)

‎exporter/http_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func TestHTTPScrapeMetricsEndpoints(t *testing.T) {
2828
defer deleteTestKeys(t, os.Getenv("TEST_PWD_REDIS_URI"))
2929

3030
csk := dbNumStrFull + "=" + url.QueryEscape(testKeys[0]) // check-single-keys
31-
css := dbNumStrFull + "=" + TestKeysStreamName // check-single-streams
31+
css := dbNumStrFull + "=" + TestKeyNameStream // check-single-streams
3232
cntk := dbNumStrFull + "=" + testKeys[0] + "*" // count-keys
3333

3434
u, err := url.Parse(os.Getenv("TEST_REDIS_URI"))
@@ -240,7 +240,7 @@ func TestSimultaneousMetricsHttpRequests(t *testing.T) {
240240
os.Getenv("TEST_REDIS6_URI"),
241241
os.Getenv("TEST_REDIS_MODULES_URI"),
242242

243-
// tile38 & Cluster need to be last in this list so we can identify them when selected, down in line 229
243+
// tile38 & Cluster need to be last in this list, so we can identify them when selected, down in line 229
244244
os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI"),
245245
os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI"),
246246
os.Getenv("TEST_TILE38_URI"),
@@ -262,7 +262,7 @@ func TestSimultaneousMetricsHttpRequests(t *testing.T) {
262262
v.Add("target", target)
263263

264264
// not appending this param for Tile38 and cluster (the last two in the list)
265-
// Tile38 & cluster don't support the SELECT command so this test will fail and spam the logs
265+
// Tile38 & cluster don't support the SELECT command, so this test will fail and spam the logs
266266
if uriIdx < len(uris)-3 {
267267
v.Add("check-single-keys", dbNumStrFull+"="+url.QueryEscape(testKeys[0]))
268268
}

‎exporter/info.go

+13-3
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,13 @@ var reMasterDirect = regexp.MustCompile(`^(master(_[0-9]+)?_(last_io_seconds_ago
3333
slave0:ip=10.254.11.1,port=6379,state=online,offset=1751844676,lag=0
3434
slave1:ip=10.254.11.2,port=6379,state=online,offset=1751844222,lag=0
3535
*/
36+
3637
var reSlave = regexp.MustCompile(`^slave\d+`)
3738

39+
const (
40+
InstanceRoleSlave = "slave"
41+
)
42+
3843
func extractVal(s string) (val float64, err error) {
3944
split := strings.Split(s, "=")
4045
if len(split) != 2 {
@@ -60,7 +65,8 @@ func extractPercentileVal(s string) (percentile float64, val float64, err error)
6065
return
6166
}
6267

63-
func (e *Exporter) extractInfoMetrics(ch chan<- prometheus.Metric, info string, dbCount int) {
68+
// returns the role of the instance we're scraping (master or slave)
69+
func (e *Exporter) extractInfoMetrics(ch chan<- prometheus.Metric, info string, dbCount int) string {
6470
keyValues := map[string]string{}
6571
handledDBs := map[string]bool{}
6672
cmdCount := map[string]uint64{}
@@ -161,8 +167,10 @@ func (e *Exporter) extractInfoMetrics(ch chan<- prometheus.Metric, info string,
161167
}
162168
}
163169

170+
instanceRole := keyValues["role"]
171+
164172
e.registerConstMetricGauge(ch, "instance_info", 1,
165-
keyValues["role"],
173+
instanceRole,
166174
keyValues["redis_version"],
167175
keyValues["redis_build_id"],
168176
keyValues["redis_mode"],
@@ -174,12 +182,14 @@ func (e *Exporter) extractInfoMetrics(ch chan<- prometheus.Metric, info string,
174182
keyValues["master_replid"],
175183
)
176184

177-
if keyValues["role"] == "slave" {
185+
if instanceRole == InstanceRoleSlave {
178186
e.registerConstMetricGauge(ch, "slave_info", 1,
179187
keyValues["master_host"],
180188
keyValues["master_port"],
181189
keyValues["slave_read_only"])
182190
}
191+
192+
return instanceRole
183193
}
184194

185195
func (e *Exporter) generateCommandLatencySummaries(ch chan<- prometheus.Metric, cmdLatencyMap map[string]map[float64]float64, cmdCount map[string]uint64, cmdSum map[string]float64) {

0 commit comments

Comments
 (0)