Merge pull request #148 from cashapp/ioberst-innodb_buffer_pool

Adding `innodb.buffer-pool` domain

Author: Areson

docs/content/metrics/domains/innodb.buffer-pool/_index.md

@@ -1,79 +1,25 @@
 ---
-title: "innodb"
+title: "innodb.buffer-pool"
 ---
 
-The `innodb` domain includes InnoDB metrics from [`INFORMATION_SCHEMA.INNODB_METRICS`](https://dev.mysql.com/doc/refman/en/information-schema-innodb-metrics-table.html).
+The `innodb.buffer-pool` domain includes InnoDB metrics from [`INFORMATION_SCHEMA.INNODB_BUFFER_POOL_STATS`](https://dev.mysql.com/doc/refman/8.4/en/information-schema-innodb-buffer-pool-stats-table.html). 
 
 {{< toc >}}
 
 ## Usage 
 
+The metric values collected are aggregated over all buffer pools.
+
 For example:
 
 ```
-mysql> SELECT * FROM innodb_metrics WHERE name='trx_rseg_history_len' LIMIT 1\G
+mysql> SELECT SUM(POOL_SIZE) POOL_SIZE, SUM(FREE_BUFFERS) FREE_BUFFERS FROM INFORMATION_SCHEMA.INNODB_BUFFER_POOL_STATS\G
 *************************** 1. row ***************************
-           NAME: trx_rseg_history_len
-      SUBSYSTEM: transaction
-          COUNT: 0
-      MAX_COUNT: 0
-      MIN_COUNT: 0
-      AVG_COUNT: NULL
-    COUNT_RESET: 0
-MAX_COUNT_RESET: 0
-MIN_COUNT_RESET: 0
-AVG_COUNT_RESET: NULL
-   TIME_ENABLED: 2021-08-17 08:24:14
-  TIME_DISABLED: NULL
-   TIME_ELAPSED: 1905927
-     TIME_RESET: NULL
-         STATUS: enabled
-           TYPE: value
-        COMMENT: Length of the TRX_RSEG_HISTORY list
+   POOL_SIZE: 95110
+   FREE_BUFFERS: 9322
 ```
 
 The exact `NAME` value is used for the Blip metric name.
-In the example above, the Blip metric name is `trx_rseg_history_len`, even though this metric means history list length (HLL).
-Metric names are unique by `SUBSYSTEM`.
-
-The [`status.global`]({{< ref "metrics/domains/status.global/" >}}) domain includes many of the same metrics because, historically, only `SHOW GLOBAL STATUS` existed.
-It's a best practice to collect InnoDB metrics with this domain and exclude them from [`status.global`]({{< ref "metrics/domains/status.global/" >}}).
-
-As a starting point, these are good InnoDB metrics to collect:
-
-```yaml
-plan:
-  collect:
-    innodb:
-      metrics:
-        # Transactions
-        - "trx_active_transactions"
-        
-        # Row locking
-        - "lock_timeouts"
-        - "lock_row_lock_current_waits"
-        - "lock_row_lock_waits"
-        - "lock_row_lock_time"
-        
-        # Page flushing
-        - "buffer_flush_adaptive_total_pages"   #  adaptive flushing
-        - "buffer_LRU_batch_flush_total_pages"  #  LRU flushing
-        - "buffer_flush_background_total_pages" #  legacy flushing
-        
-        # Transaction log utilization (%)
-        - "log_lsn_checkpoint_age"     # checkpoint age
-        - "log_max_modified_age_async" # async flush point
-        
-        # Transaction log -> storage waits
-        - "innodb_os_log_pending_writes"
-        - "innodb_log_waits"
-        
-        # History List Length (HLL)
-        - "trx_rseg_history_len"
-        
-        # Deadlocks
-        - "lock_deadlocks"
-``` 
 
 ## Derived Metrics
 
@@ -85,33 +31,27 @@ None.
 
 |Value|Default|Description|
 |-----|-------|-----------|
-|yes  | |Collect _all_ 300+ metrics (not recommended)|
+|yes  | |Collect _all_ 30+ metrics (not recommended)|
 |no   |&check;|Collect only metrics listed in the plan|
-|enabled| |Collect metrics that are enabled by MySQL (`WHERE status='enabled'`)|
 
 ## Group Keys
 
 None.
 
 ## Meta
 
-|Key|Value|
-|---|-----|
-|`subsystem`|`SUBSYSTEM` column|
-
-Technically InnoDB metric names are unique by subsystem, but currently they're unique for the whole table.
-It's probably safe to graph them by metric name alone, but if there's a name collision in the future, one metrics will be lost when reporting.
+None.
 
 ## Error Policies
 
 None.
 
 ## MySQL Config
 
-See [17.15.6 InnoDB INFORMATION_SCHEMA Metrics Table](https://dev.mysql.com/doc/refman/en/innodb-information-schema-metrics-table.html).
+None.
 
 ## Changelog
 
 |Blip Version|Change|
 |------------|------|
-|v1.0.0      |Domain added|
+|TBD      |Domain added|

docs/content/metrics/domains/innodb/_index.md

@@ -29,6 +29,7 @@ The rest are reserved for future use.
 |gr|MySQL Group Replication||
 |host|Host (client)||
 |[`innodb`](domains#innodb)|InnoDB metrics [`INFORMATION_SCHEMA.INNODB_METRICS`](https://dev.mysql.com/doc/refman/en/information-schema-innodb-metrics-table.html)|v1.0.0|
+|[`innodb.buffer-pool`](domains#innodbbuffer-pool)|InnoDB buffer pool metrics [`INFORMATION_SCHEMA.INNODB_BUFFER_POOL_STAT`](https://dev.mysql.com/doc/refman/8.4/en/information-schema-innodb-buffer-pool-stats-table.html)|TBD|
 |innodb.mutex|InnoDB mutexes `SHOW ENGINE INNODB MUTEX`||
 |mariadb|MariaDB enhancements||
 |ndb|MySQL NDB Cluster||

docs/content/metrics/quick-ref.md

@@ -8,21 +8,22 @@ import (
 	"sync"
 
 	"github.com/cashapp/blip"
-	"github.com/cashapp/blip/metrics/aws.rds"
+	awsrds "github.com/cashapp/blip/metrics/aws.rds"
 	"github.com/cashapp/blip/metrics/innodb"
+	innodbbufferpool "github.com/cashapp/blip/metrics/innodb.buffer-pool"
 	"github.com/cashapp/blip/metrics/percona"
-	"github.com/cashapp/blip/metrics/query.response-time"
+	queryresponsetime "github.com/cashapp/blip/metrics/query.response-time"
 	"github.com/cashapp/blip/metrics/repl"
-	"github.com/cashapp/blip/metrics/repl.lag"
-	"github.com/cashapp/blip/metrics/size.binlog"
-	"github.com/cashapp/blip/metrics/size.database"
-	"github.com/cashapp/blip/metrics/size.table"
-	"github.com/cashapp/blip/metrics/status.global"
+	repllag "github.com/cashapp/blip/metrics/repl.lag"
+	sizebinlog "github.com/cashapp/blip/metrics/size.binlog"
+	sizedatabase "github.com/cashapp/blip/metrics/size.database"
+	sizetable "github.com/cashapp/blip/metrics/size.table"
+	statusglobal "github.com/cashapp/blip/metrics/status.global"
 	"github.com/cashapp/blip/metrics/stmt.current"
 	"github.com/cashapp/blip/metrics/tls"
 	"github.com/cashapp/blip/metrics/trx"
-	"github.com/cashapp/blip/metrics/var.global"
-	"github.com/cashapp/blip/metrics/wait.io.table"
+	varglobal "github.com/cashapp/blip/metrics/var.global"
+	waitiotable "github.com/cashapp/blip/metrics/wait.io.table"
 )
 
 // Register registers a factory that makes one or more collector by domain name.
@@ -260,6 +261,8 @@ func (f *factory) Make(domain string, args blip.CollectorFactoryArgs) (blip.Coll
 		return awsrds.NewRDS(awsrds.NewCloudWatchClient(awsConfig)), nil
 	case "innodb":
 		return innodb.NewInnoDB(args.DB), nil
+	case "innodb.buffer-pool":
+		return innodbbufferpool.NewBufferPoolStats(args.DB), nil
 	case "percona.response-time":
 		return percona.NewQRT(args.DB), nil
 	case "query.response-time":
@@ -295,6 +298,7 @@ func (f *factory) Make(domain string, args blip.CollectorFactoryArgs) (blip.Coll
 var builtinCollectors = []string{
 	"aws.rds",
 	"innodb",
+	"innodb.buffer-pool",
 	"percona.response-time",
 	"query.response-time",
 	"repl",

metrics/factory.go

@@ -0,0 +1,182 @@
+// Copyright 2024 Block, Inc.
+
+package innodbbufferpool
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"github.com/cashapp/blip"
+	"github.com/cashapp/blip/sqlutil"
+)
+
+// Table collects buffer pool data from information_schema.innodb_buffer_pool_stats.
+// https://dev.mysql.com/doc/refman/8.4/en/information-schema-innodb-buffer-pool-stats-table.html
+
+const (
+	DOMAIN  = "innodb.buffer-pool"
+	OPT_ALL = "all"
+
+	BASE_QUERY = "SELECT %s FROM information_schema.innodb_buffer_pool_stats"
+)
+
+var (
+	columnNames = []string{
+		"pool_size",
+		"free_buffers",
+		"database_pages",
+		"old_database_pages",
+		"modified_database_pages",
+		"pending_decompress",
+		"pending_reads",
+		"pending_flush_lru",
+		"pending_flush_list",
+		"pages_made_young",
+		"pages_not_made_young",
+		"pages_made_young_rate",
+		"pages_made_not_young_rate",
+		"number_pages_read",
+		"number_pages_created",
+		"number_pages_written",
+		"pages_read_rate",
+		"pages_create_rate",
+		"pages_written_rate",
+		"number_pages_get",
+		"hit_rate",
+		"young_make_per_thousand_get",
+		"not_young_make_per_thousand_get",
+		"number_pages_read_ahead",
+		"number_read_ahead_evicted",
+		"read_ahead_rate",
+		"read_ahead_evicted_rate",
+		"lru_io_total",
+		"lru_io_current",
+		"uncompress_total",
+		"uncompress_current",
+	}
+
+	columnSum map[string]string
+)
+
+func init() {
+	columnSum = make(map[string]string, len(columnNames))
+	for _, name := range columnNames {
+		columnSum[name] = fmt.Sprintf("SUM(%s) as %s", name, name)
+	}
+}
+
+// BufferPoolStats collections from the information_schema.innodb_buffer_pool_stats table.
+type BufferPoolStats struct {
+	db    *sql.DB
+	query map[string]string
+}
+
+// Verify collector implements blip.Collector interface.
+var _ blip.Collector = &BufferPoolStats{}
+
+// NewTable makes a new Table collector,
+func NewBufferPoolStats(db *sql.DB) *BufferPoolStats {
+	return &BufferPoolStats{
+		db:    db,
+		query: map[string]string{},
+	}
+}
+
+// Domain returns the Blip metric domain name (DOMAIN const).
+func (t *BufferPoolStats) Domain() string {
+	return DOMAIN
+}
+
+// Help returns the output for blip --print-domains.
+func (t BufferPoolStats) Help() blip.CollectorHelp {
+	return blip.CollectorHelp{
+		Domain:      DOMAIN,
+		Description: "Buffer Pool Stats (summed over all pools)",
+		Options: map[string]blip.CollectorHelpOption{
+			OPT_ALL: {
+				Name:    OPT_ALL,
+				Desc:    "Collect metrics from all columns in the information_schema.innodb_buffer_pool_stats table.",
+				Default: "no",
+				Values: map[string]string{
+					"yes": "All metrics (ignore metrics list)",
+					"no":  "Specified metrics",
+				},
+			},
+		},
+	}
+}
+
+// Prepare prepares the collector for the given plan.
+func (c BufferPoolStats) Prepare(ctx context.Context, plan blip.Plan) (func(), error) {
+LEVEL:
+	for _, level := range plan.Levels {
+		dom, ok := level.Collect[DOMAIN]
+		if !ok {
+			continue LEVEL // not collected at this level
+		}
+
+		sumFields := make([]string, 0, len(dom.Metrics))
+		all := strings.ToLower(dom.Options[OPT_ALL])
+		switch all {
+		case "all":
+			for _, column := range columnNames {
+				sumFields = append(sumFields, columnSum[column])
+			}
+		default:
+			for _, metric := range dom.Metrics {
+				value, ok := columnSum[metric]
+				if !ok {
+					return nil, fmt.Errorf("invalid metric %q", metric)
+				}
+
+				sumFields = append(sumFields, value)
+			}
+		}
+
+		c.query[level.Name] = fmt.Sprintf("SELECT %s FROM information_schema.innodb_buffer_pool_stats", strings.Join(sumFields, ", "))
+		blip.Debug("%s: innodb metrics at %s: %s", plan.MonitorId, level.Name, c.query[level.Name])
+	}
+	return nil, nil
+}
+
+func (t BufferPoolStats) Collect(ctx context.Context, levelName string) ([]blip.MetricValue, error) {
+	o, ok := t.query[levelName]
+	if !ok {
+		return nil, nil
+	}
+
+	results, err := sqlutil.RowToTypedMap[float64](ctx, t.db, o)
+	if err != nil {
+		return nil, err
+	}
+
+	metrics := make([]blip.MetricValue, 0, len(results))
+	for name, value := range results {
+		m := blip.MetricValue{
+			Name:  name,
+			Type:  blip.CUMULATIVE_COUNTER,
+			Value: value,
+		}
+
+		if deltas[name] {
+			m.Type = blip.DELTA_COUNTER
+		}
+
+		metrics = append(metrics, m)
+	}
+
+	return metrics, nil
+}
+
+// deltas is a list of known delta metrics in information_schema.innodb_buffer_pool_stats.
+var deltas = map[string]bool{
+	"pages_made_young_rate":     true,
+	"pages_made_not_young_rate": true,
+	"pages_read_rate":           true,
+	"pages_create_rate":         true,
+	"pages_written_rate":        true,
+	"read_ahead_rate":           true,
+	"read_ahead_evicted_rate":   true,
+}