Skip to content

Commit 1c5387d

Browse files
daniel-nichterdaniel-nichter
authored
Merge branch 'main' into dn/mysql84
2 parents f45a222 + 2916197 commit 1c5387d

File tree

4 files changed

+278
-0
lines changed

4 files changed

+278
-0
lines changed

‎metrics/autoinc/autoinc.go‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
// Copyright 2024 Block, Inc.
2+
3+
package autoinc
4+
5+
import (
6+
"context"
7+
"database/sql"
8+
"strings"
9+
10+
"github.com/cashapp/blip"
11+
)
12+
13+
const (
14+
DOMAIN = "autoinc"
15+
16+
OPT_EXCLUDE = "exclude"
17+
OPT_INCLUDE = "include"
18+
)
19+
20+
// AutoInc collects auto-increment utilization for the autoinc domain.
21+
// https://dev.mysql.com/doc/refman/8.0/en/sys-schema-auto-increment-columns.html
22+
type AutoInc struct {
23+
db *sql.DB
24+
// --
25+
query map[string]string
26+
params map[string][]interface{}
27+
}
28+
29+
// Verify collector implements blip.Collector interface.
30+
var _ blip.Collector = &AutoInc{}
31+
32+
// NewAutoIncrement makes a new AutoIncrement collector,
33+
func NewAutoInc(db *sql.DB) *AutoInc {
34+
return &AutoInc{
35+
db: db,
36+
query: map[string]string{},
37+
params: map[string][]interface{}{},
38+
}
39+
}
40+
41+
// Domain returns the Blip metric domain name (DOMAIN const).
42+
func (t *AutoInc) Domain() string {
43+
return DOMAIN
44+
}
45+
46+
// Help returns the output for blip --print-domains.
47+
func (t *AutoInc) Help() blip.CollectorHelp {
48+
return blip.CollectorHelp{
49+
Domain: DOMAIN,
50+
Description: "Auto Increment Utilization",
51+
Options: map[string]blip.CollectorHelpOption{
52+
OPT_INCLUDE: {
53+
Name: OPT_INCLUDE,
54+
Desc: "Comma-separated list of database or table names to include (overrides option " + OPT_EXCLUDE + ")",
55+
},
56+
OPT_EXCLUDE: {
57+
Name: OPT_EXCLUDE,
58+
Desc: "Comma-separated list of database or table names to exclude (ignored if " + OPT_INCLUDE + " is set)",
59+
Default: "mysql.*,information_schema.*,performance_schema.*,sys.*",
60+
},
61+
},
62+
Groups: []blip.CollectorKeyValue{
63+
{Key: "db", Value: "the database name for the corresponding auto increment value"},
64+
{Key: "tbl", Value: "the table name for the corresponding auto increment value"},
65+
{Key: "col", Value: "the column name for the corresponding auto increment value"},
66+
{Key: "data_type", Value: "the data type of the column"},
67+
},
68+
Metrics: []blip.CollectorMetric{
69+
{
70+
Name: "usage",
71+
Type: blip.GAUGE,
72+
Desc: "The percentage of the auto increment range used",
73+
},
74+
},
75+
}
76+
}
77+
78+
// Prepare prepares the collector for the given plan.
79+
func (t *AutoInc) Prepare(ctx context.Context, plan blip.Plan) (func(), error) {
80+
LEVEL:
81+
for _, level := range plan.Levels {
82+
dom, ok := level.Collect[DOMAIN]
83+
if !ok {
84+
continue LEVEL // not collected in this level
85+
}
86+
if dom.Options == nil {
87+
dom.Options = make(map[string]string)
88+
}
89+
if _, ok := dom.Options[OPT_EXCLUDE]; !ok {
90+
dom.Options[OPT_EXCLUDE] = "mysql.*,information_schema.*,performance_schema.*,sys.*"
91+
}
92+
93+
q, params, err := AutoIncrementQuery(dom.Options)
94+
if err != nil {
95+
return nil, err
96+
}
97+
t.query[level.Name] = q
98+
t.params[level.Name] = params
99+
}
100+
return nil, nil
101+
}
102+
103+
func (t *AutoInc) Collect(ctx context.Context, levelName string) ([]blip.MetricValue, error) {
104+
q, ok := t.query[levelName]
105+
if !ok {
106+
return nil, nil
107+
}
108+
109+
rows, err := t.db.QueryContext(ctx, q, t.params[levelName]...)
110+
if err != nil {
111+
return nil, err
112+
}
113+
defer rows.Close()
114+
115+
var (
116+
metrics []blip.MetricValue
117+
dbName string
118+
tblName string
119+
colName string
120+
colType string
121+
isUnsigned bool
122+
autoincVal int64
123+
)
124+
125+
for rows.Next() {
126+
if err = rows.Scan(&dbName, &tblName, &colName, &colType, &isUnsigned, &autoincVal); err != nil {
127+
return nil, err
128+
}
129+
130+
var maxSize uint64
131+
colType = strings.ToLower(colType)
132+
switch colType {
133+
case "tinyint":
134+
maxSize = 255
135+
case "smallint":
136+
maxSize = 65535
137+
case "mediumint":
138+
maxSize = 16777215
139+
case "int":
140+
maxSize = 4294967295
141+
case "bigint":
142+
maxSize = 18446744073709551615
143+
default:
144+
// unknown type, skip
145+
continue
146+
}
147+
148+
if !isUnsigned {
149+
maxSize = maxSize >> 1
150+
} else {
151+
colType = colType + " unsigned"
152+
}
153+
154+
m := blip.MetricValue{
155+
Name: "usage",
156+
Type: blip.GAUGE,
157+
Group: map[string]string{"db": dbName, "tbl": tblName, "col": colName, "data_type": colType},
158+
Value: float64(autoincVal) / float64(maxSize),
159+
}
160+
metrics = append(metrics, m)
161+
}
162+
163+
return metrics, err
164+
}

‎metrics/autoinc/query.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Copyright 2024 Block, Inc.
2+
3+
package autoinc
4+
5+
import (
6+
"strings"
7+
)
8+
9+
const (
10+
base = `SELECT C.TABLE_SCHEMA, C.TABLE_NAME, C.COLUMN_NAME, C.DATA_TYPE, (locate('unsigned', C.COLUMN_TYPE) > 0) AS is_unsigned, T.AUTO_INCREMENT
11+
FROM information_schema.COLUMNS C
12+
JOIN information_schema.TABLES T
13+
ON C.TABLE_SCHEMA = T.TABLE_SCHEMA
14+
AND C.TABLE_NAME = T.TABLE_NAME
15+
WHERE T.TABLE_TYPE = 'BASE TABLE'
16+
AND C.EXTRA = 'auto_increment'
17+
AND T.AUTO_INCREMENT IS NOT NULL`
18+
)
19+
20+
func AutoIncrementQuery(set map[string]string) (string, []interface{}, error) {
21+
var where string
22+
var params []interface{}
23+
if include := set[OPT_INCLUDE]; include != "" {
24+
where, params = setWhere(strings.Split(set[OPT_INCLUDE], ","), true)
25+
} else {
26+
where, params = setWhere(strings.Split(set[OPT_EXCLUDE], ","), false)
27+
}
28+
return base + where, params, nil
29+
}
30+
31+
func setWhere(tables []string, isInclude bool) (string, []interface{}) {
32+
where := " AND ("
33+
if !isInclude {
34+
where = where + "NOT "
35+
}
36+
var params []interface{} = make([]interface{}, 0)
37+
for i, excludeTable := range tables {
38+
if strings.Contains(excludeTable, ".") {
39+
dbAndTable := strings.Split(excludeTable, ".")
40+
db := dbAndTable[0]
41+
table := dbAndTable[1]
42+
if table == "*" {
43+
where = where + "(C.TABLE_SCHEMA = ?)"
44+
params = append(params, db)
45+
} else {
46+
where = where + "(C.TABLE_SCHEMA = ? AND C.TABLE_NAME = ?)"
47+
params = append(params, db, table)
48+
}
49+
} else {
50+
where = where + "(C.TABLE_NAME = ?)"
51+
params = append(params, excludeTable)
52+
}
53+
if i != (len(tables) - 1) {
54+
if isInclude {
55+
where = where + " OR "
56+
} else {
57+
where = where + " AND NOT "
58+
}
59+
}
60+
}
61+
where = where + ")"
62+
return where, params
63+
}

‎metrics/autoinc/query_test.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
// Copyright 2024 Block, Inc.
2+
3+
package autoinc
4+
5+
import (
6+
"testing"
7+
8+
"github.com/go-test/deep"
9+
)
10+
11+
func TestAutoIncrementQuery(t *testing.T) {
12+
// All defaults
13+
opts := map[string]string{
14+
OPT_EXCLUDE: "mysql.*,information_schema.*,performance_schema.*,sys.*",
15+
}
16+
got, params, err := AutoIncrementQuery(opts)
17+
expect := base + " AND (NOT (C.TABLE_SCHEMA = ?) AND NOT (C.TABLE_SCHEMA = ?) AND NOT (C.TABLE_SCHEMA = ?) AND NOT (C.TABLE_SCHEMA = ?))"
18+
if err != nil {
19+
t.Error(err)
20+
}
21+
if got != expect {
22+
t.Errorf("got:\n%s\nexpect:\n%s\n", got, expect)
23+
}
24+
25+
expectedParams := []interface{}{"mysql", "information_schema", "performance_schema", "sys"}
26+
if diff := deep.Equal(params, expectedParams); diff != nil {
27+
t.Error(diff)
28+
}
29+
30+
// Exclude schemas, mysql, and sys
31+
opts = map[string]string{
32+
OPT_INCLUDE: "test_table,sys.*,information_schema.XTRADB_ZIP_DICT",
33+
}
34+
got, params, err = AutoIncrementQuery(opts)
35+
expect = base + " AND ((C.TABLE_NAME = ?) OR (C.TABLE_SCHEMA = ?) OR (C.TABLE_SCHEMA = ? AND C.TABLE_NAME = ?))"
36+
if err != nil {
37+
t.Error(err)
38+
}
39+
if got != expect {
40+
t.Errorf("got:\n%s\nexpect:\n%s\n", got, expect)
41+
}
42+
43+
expectedParams = []interface{}{"test_table", "sys", "information_schema", "XTRADB_ZIP_DICT"}
44+
if diff := deep.Equal(params, expectedParams); diff != nil {
45+
t.Error(diff)
46+
}
47+
}

‎metrics/factory.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"sync"
99

1010
"github.com/cashapp/blip"
11+
"github.com/cashapp/blip/metrics/autoinc"
1112
awsrds "github.com/cashapp/blip/metrics/aws.rds"
1213
"github.com/cashapp/blip/metrics/innodb"
1314
innodbbufferpool "github.com/cashapp/blip/metrics/innodb.buffer-pool"
@@ -246,6 +247,8 @@ func InitFactory(factories blip.Factories) {
246247
// that makes the built-in collectors: status.global, var.global, and so on.
247248
func (f *factory) Make(domain string, args blip.CollectorFactoryArgs) (blip.Collector, error) {
248249
switch domain {
250+
case "autoinc":
251+
return autoinc.NewAutoInc(args.DB), nil
249252
case "aws.rds":
250253
if args.Validate {
251254
return awsrds.NewRDS(nil), nil
@@ -296,6 +299,7 @@ func (f *factory) Make(domain string, args blip.CollectorFactoryArgs) (blip.Coll
296299
// List of built-in collectors. To add one, add its domain name here, and add
297300
// the same domain in the switch statement above (in factory.Make).
298301
var builtinCollectors = []string{
302+
"autoinc",
299303
"aws.rds",
300304
"innodb",
301305
"innodb.buffer-pool",

0 commit comments

Comments
 (0)