diff --git a/pkg/metrics/client_go_adapter.go b/pkg/metrics/client_go_adapter.go index ff28998c44..e7d3afe878 100644 --- a/pkg/metrics/client_go_adapter.go +++ b/pkg/metrics/client_go_adapter.go @@ -18,17 +18,34 @@ package metrics import ( "context" + "math" + "net/url" + "time" "github.com/prometheus/client_golang/prometheus" clientmetrics "k8s.io/client-go/tools/metrics" ) -// this file contains setup logic to initialize the myriad of places -// that client-go registers metrics. We copy the names and formats -// from Kubernetes so that we match the core controllers. - var ( - // client metrics. + // requestLatency is a Prometheus Histogram metric type partitioned by + // "verb", and "host" labels. It is used for the rest client latency metrics. + requestLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "rest_client_request_duration_seconds", + Help: "Request latency in seconds. Broken down by verb, and host.", + Buckets: []float64{math.Inf(+1)}, // Intentionally using minimum buckets for better performance / lower cardinality. + }, + []string{"verb", "host"}, + ) + + rateLimiterLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "rest_client_rate_limiter_duration_seconds", + Help: "Client side rate limiter latency in seconds. Broken down by verb, and host.", + Buckets: []float64{math.Inf(+1)}, // Intentionally using minimum buckets for better performance / lower cardinality. + }, + []string{"verb", "host"}, + ) requestResult = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -37,6 +54,14 @@ var ( }, []string{"code", "method", "host"}, ) + + requestRetry = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rest_client_request_retries_total", + Help: "Number of request retries, partitioned by status code, verb, and host.", + }, + []string{"code", "verb", "host"}, + ) ) func init() { @@ -46,11 +71,17 @@ func init() { // registerClientMetrics sets up the client latency metrics from client-go. func registerClientMetrics() { // register the metrics with our registry + Registry.MustRegister(requestLatency) + Registry.MustRegister(rateLimiterLatency) Registry.MustRegister(requestResult) + Registry.MustRegister(requestRetry) // register the metrics with client-go clientmetrics.Register(clientmetrics.RegisterOpts{ - RequestResult: &resultAdapter{metric: requestResult}, + RequestLatency: &LatencyAdapter{metric: requestLatency}, + RateLimiterLatency: &LatencyAdapter{metric: rateLimiterLatency}, + RequestResult: &resultAdapter{metric: requestResult}, + RequestRetry: &retryAdapter{requestRetry}, }) } @@ -62,6 +93,16 @@ func registerClientMetrics() { // copied (more-or-less directly) from k8s.io/kubernetes setup code // (which isn't anywhere in an easily-importable place). +// LatencyAdapter implements LatencyMetric. +type LatencyAdapter struct { + metric *prometheus.HistogramVec +} + +// Observe increments the request latency metric for the given verb/URL. +func (l *LatencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { + l.metric.WithLabelValues(verb, u.String()).Observe(latency.Seconds()) +} + type resultAdapter struct { metric *prometheus.CounterVec } @@ -69,3 +110,11 @@ type resultAdapter struct { func (r *resultAdapter) Increment(_ context.Context, code, method, host string) { r.metric.WithLabelValues(code, method, host).Inc() } + +type retryAdapter struct { + metric *prometheus.CounterVec +} + +func (r *retryAdapter) IncrementRetry(_ context.Context, code, method, host string) { + r.metric.WithLabelValues(code, method, host).Inc() +}