Time framed thread-pool utilization

benchmarks/src/main/java/org/elasticsearch/benchmark/common/util/concurrent/ThreadPoolUtilizationBenchmark.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.common.util.concurrent;
+
+import org.elasticsearch.common.util.concurrent.TaskExecutionTimeTrackingEsThreadPoolExecutor;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Group;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+@Threads(Threads.MAX)
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Measurement(iterations = 5, time = 1, timeUnit = TimeUnit.MINUTES)
+@State(Scope.Benchmark)
+@Fork(1)
+public class ThreadPoolUtilizationBenchmark {
+
+    @Param({ "10000" })
+    private int callIntervalTicks;
+
+    /**
+     * This makes very little difference, all the overhead is in the synchronization
+     */
+    @Param({ "1000" })
+    private int frameDurationMs;
+
+    @Param({ "10000" })
+    private int reportingDurationMs;
+
+    private TaskExecutionTimeTrackingEsThreadPoolExecutor.FramedTimeTracker timeTracker;
+
+    @Setup
+    public void setup() {
+        timeTracker = new TaskExecutionTimeTrackingEsThreadPoolExecutor.FramedTimeTracker(
+            Duration.ofMillis(reportingDurationMs).toNanos(),
+            Duration.ofMillis(frameDurationMs).toNanos(),
+            System::nanoTime
+        );
+    }
+
+    @Benchmark
+    public void baseline() {
+        Blackhole.consumeCPU(callIntervalTicks);
+    }
+
+    @Group("StartAndEnd")
+    @Benchmark
+    public void startAndStopTasks() {
+        timeTracker.startTask();
+        Blackhole.consumeCPU(callIntervalTicks);
+        timeTracker.endTask();
+    }
+}

docs/changelog/131898.yaml

@@ -0,0 +1,5 @@
+pr: 131898
+summary: Time framed thread-pool utilization
+area: Allocation
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.node.Node;
 
+import java.time.Duration;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.AbstractExecutorService;
@@ -576,80 +577,80 @@ public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) {
         }
     }
 
-    public static class TaskTrackingConfig {
+    /**
+     * @param trackExecutionTime Whether to track execution stats
+     * @param trackUtilization Enables thread-pool utilization metrics
+     * @param utilizationReportingInterval When utilization is enabled, specifies interval for reporting utilization, default 30 seconds
+     * @param utilizationSamplingInterval When utilization is enabled, specifies sample interval, default 1 second
+     * @param trackOngoingTasks Whether to track ongoing task execution time, not just finished tasks
+     * @param trackMaxQueueLatency Whether to track max queue latency
+     * @param executionTimeEwmaAlpha The alpha seed for execution time EWMA (ExponentiallyWeightedMovingAverage)
+     */
+    public record TaskTrackingConfig(
+        boolean trackExecutionTime,
+        boolean trackUtilization,
+        Duration utilizationReportingInterval,
+        Duration utilizationSamplingInterval,
+        boolean trackOngoingTasks,
+        boolean trackMaxQueueLatency,
+        double executionTimeEwmaAlpha
+    ) {
+
         // This is a random starting point alpha.
         public static final double DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST = 0.3;
-
-        private final boolean trackExecutionTime;
-        private final boolean trackOngoingTasks;
-        private final boolean trackMaxQueueLatency;
-        private final double executionTimeEwmaAlpha;
+        public static final Duration DEFAULT_UTILIZATION_INTERVAL = Duration.ofSeconds(30);
+        public static final Duration DEFAULT_UTILIZATION_SAMPLING_INTERVAL = Duration.ofSeconds(1);
 
         public static final TaskTrackingConfig DO_NOT_TRACK = new TaskTrackingConfig(
             false,
             false,
+            DEFAULT_UTILIZATION_INTERVAL,
+            DEFAULT_UTILIZATION_SAMPLING_INTERVAL,
+            false,
             false,
             DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST
         );
+
         public static final TaskTrackingConfig DEFAULT = new TaskTrackingConfig(
             true,
+            true,
+            DEFAULT_UTILIZATION_INTERVAL,
+            DEFAULT_UTILIZATION_SAMPLING_INTERVAL,
             false,
             false,
             DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST
         );
 
-        /**
-         * @param trackExecutionTime Whether to track execution stats
-         * @param trackOngoingTasks Whether to track ongoing task execution time, not just finished tasks
-         * @param trackMaxQueueLatency Whether to track max queue latency.
-         * @param executionTimeEWMAAlpha The alpha seed for execution time EWMA (ExponentiallyWeightedMovingAverage).
-         */
-        private TaskTrackingConfig(
-            boolean trackExecutionTime,
-            boolean trackOngoingTasks,
-            boolean trackMaxQueueLatency,
-            double executionTimeEWMAAlpha
-        ) {
-            this.trackExecutionTime = trackExecutionTime;
-            this.trackOngoingTasks = trackOngoingTasks;
-            this.trackMaxQueueLatency = trackMaxQueueLatency;
-            this.executionTimeEwmaAlpha = executionTimeEWMAAlpha;
-        }
-
-        public boolean trackExecutionTime() {
-            return trackExecutionTime;
-        }
-
-        public boolean trackOngoingTasks() {
-            return trackOngoingTasks;
-        }
-
-        public boolean trackMaxQueueLatency() {
-            return trackMaxQueueLatency;
-        }
-
-        public double getExecutionTimeEwmaAlpha() {
-            return executionTimeEwmaAlpha;
-        }
-
         public static Builder builder() {
             return new Builder();
         }
 
         public static class Builder {
             private boolean trackExecutionTime = false;
+            private boolean trackUtilization = false;
             private boolean trackOngoingTasks = false;
             private boolean trackMaxQueueLatency = false;
             private double ewmaAlpha = DEFAULT_EXECUTION_TIME_EWMA_ALPHA_FOR_TEST;
+            private Duration utilizationInterval = DEFAULT_UTILIZATION_INTERVAL;
+            private Duration utilizationSamplingInterval = DEFAULT_UTILIZATION_SAMPLING_INTERVAL;
 
             public Builder() {}
 
             public Builder trackExecutionTime(double alpha) {
                 trackExecutionTime = true;
+                trackUtilization = true;
                 ewmaAlpha = alpha;
                 return this;
             }
 
+            public Builder trackUtilization(Duration interval, Duration samplingInterval) {
+                assert interval.dividedBy(samplingInterval) > 0 : "interval should be same or larger than sampling interval";
+                trackUtilization = true;
+                utilizationInterval = interval;
+                utilizationSamplingInterval = samplingInterval;
+                return this;
+            }
+
             public Builder trackOngoingTasks() {
                 trackOngoingTasks = true;
                 return this;
@@ -661,7 +662,15 @@ public Builder trackMaxQueueLatency() {
             }
 
             public TaskTrackingConfig build() {
-                return new TaskTrackingConfig(trackExecutionTime, trackOngoingTasks, trackMaxQueueLatency, ewmaAlpha);
+                return new TaskTrackingConfig(
+                    trackExecutionTime,
+                    trackUtilization,
+                    utilizationInterval,
+                    utilizationSamplingInterval,
+                    trackOngoingTasks,
+                    trackMaxQueueLatency,
+                    ewmaAlpha
+                );
             }
         }
     }