Constant stack size (#2688)

* Inline engine stages.
Apply AggressiveOptimization to engine methods.

* Update src/BenchmarkDotNet/Engines/Engine.cs

Co-authored-by: Corniel Nobel <corniel@gmail.com>

* Refactor to use `IEngineStageEvaluator` for constant instruction location and simpler engine code.

* Refactored according to PR comments.

* Fix measurementsForStatistics.

---------

Co-authored-by: Corniel Nobel <corniel@gmail.com>

Author: timcassell

src/BenchmarkDotNet/Engines/Engine.cs

@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Globalization;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Mathematics;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
 using JetBrains.Annotations;
@@ -15,8 +18,6 @@ namespace BenchmarkDotNet.Engines
     [UsedImplicitly]
     public class Engine : IEngine
     {
-        public const int MinInvokeCount = 4;
-
         [PublicAPI] public IHost Host { get; }
         [PublicAPI] public Action<long> WorkloadAction { get; }
         [PublicAPI] public Action Dummy1Action { get; }
@@ -41,9 +42,6 @@ public class Engine : IEngine
         private bool MemoryRandomization { get; }
 
         private readonly List<Measurement> jittingMeasurements = new (10);
-        private readonly EnginePilotStage pilotStage;
-        private readonly EngineWarmupStage warmupStage;
-        private readonly EngineActualStage actualStage;
         private readonly bool includeExtraStats;
         private readonly Random random;
 
@@ -79,10 +77,6 @@ internal Engine(
             EvaluateOverhead = targetJob.ResolveValue(AccuracyMode.EvaluateOverheadCharacteristic, Resolver);
             MemoryRandomization = targetJob.ResolveValue(RunMode.MemoryRandomizationCharacteristic, Resolver);
 
-            warmupStage = new EngineWarmupStage(this);
-            pilotStage = new EnginePilotStage(this);
-            actualStage = new EngineActualStage(this);
-
             random = new Random(12345); // we are using constant seed to try to get repeatable results
         }
 
@@ -102,6 +96,9 @@ public void Dispose()
             }
         }
 
+        // AggressiveOptimization forces the method to go straight to tier1 JIT, and will never be re-jitted,
+        // eliminating tiered JIT as a potential variable in measurements.
+        [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
         public RunResults Run()
         {
             var measurements = new List<Measurement>();
@@ -112,30 +109,34 @@ public RunResults Run()
             if (EngineEventSource.Log.IsEnabled())
                 EngineEventSource.Log.BenchmarkStart(BenchmarkName);
 
-            if (Strategy != RunStrategy.ColdStart)
-            {
-                if (Strategy != RunStrategy.Monitoring)
-                {
-                    var pilotStageResult = pilotStage.Run();
-                    invokeCount = pilotStageResult.PerfectInvocationCount;
-                    measurements.AddRange(pilotStageResult.Measurements);
-
-                    if (EvaluateOverhead)
-                    {
-                        measurements.AddRange(warmupStage.RunOverhead(invokeCount, UnrollFactor));
-                        measurements.AddRange(actualStage.RunOverhead(invokeCount, UnrollFactor));
-                    }
+            // Enumerate the stages and run iterations in a loop to ensure each benchmark invocation is called with a constant stack size.
+            // #1120
+            foreach (var stage in EngineStage.EnumerateStages(this, Strategy, EvaluateOverhead))
+            {
+                if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload)
+                {
+                    Host.BeforeMainRun();
+                }
+
+                var stageMeasurements = stage.GetMeasurementList();
+                // 1-based iterationIndex
+                int iterationIndex = 1;
+                while (stage.GetShouldRunIteration(stageMeasurements, ref invokeCount))
+                {
+                    var measurement = RunIteration(new IterationData(stage.Mode, stage.Stage, iterationIndex, invokeCount, UnrollFactor));
+                    stageMeasurements.Add(measurement);
+                    ++iterationIndex;
+                }
+                measurements.AddRange(stageMeasurements);
+
+                WriteLine();
+
+                if (stage.Stage == IterationStage.Actual && stage.Mode == IterationMode.Workload)
+                {
+                    Host.AfterMainRun();
                 }
-
-                measurements.AddRange(warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy));
             }
 
-            Host.BeforeMainRun();
-
-            measurements.AddRange(actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring));
-
-            Host.AfterMainRun();
-
             (GcStats workGcHasDone, ThreadingStats threadingStats, double exceptionFrequency) = includeExtraStats
                 ? GetExtraStats(new IterationData(IterationMode.Workload, IterationStage.Actual, 0, invokeCount, UnrollFactor))
                 : (GcStats.Empty, ThreadingStats.Empty, 0);
@@ -148,11 +149,15 @@ public RunResults Run()
             return new RunResults(measurements, outlierMode, workGcHasDone, threadingStats, exceptionFrequency);
         }
 
+        [MethodImpl(CodeGenHelper.AggressiveOptimizationOption)]
         public Measurement RunIteration(IterationData data)
         {
             // Initialization
             long invokeCount = data.InvokeCount;
             int unrollFactor = data.UnrollFactor;
+            if (invokeCount % unrollFactor != 0)
+                throw new ArgumentOutOfRangeException(nameof(data), $"InvokeCount({invokeCount}) should be a multiple of UnrollFactor({unrollFactor}).");
+
             long totalOperations = invokeCount * OperationsPerInvoke;
             bool isOverhead = data.IterationMode == IterationMode.Overhead;
             bool randomizeMemory = !isOverhead && MemoryRandomization;
@@ -167,7 +172,7 @@ public Measurement RunIteration(IterationData data)
                 EngineEventSource.Log.IterationStart(data.IterationMode, data.IterationStage, totalOperations);
 
             var clockSpan = randomizeMemory
-                ? MeasureWithRandomMemory(action, invokeCount / unrollFactor)
+                ? MeasureWithRandomStack(action, invokeCount / unrollFactor)
                 : Measure(action, invokeCount / unrollFactor);
 
             if (EngineEventSource.Log.IsEnabled())
@@ -193,8 +198,8 @@ public Measurement RunIteration(IterationData data)
         // This is in a separate method, because stackalloc can affect code alignment,
         // resulting in unexpected measurements on some AMD cpus,
         // even if the stackalloc branch isn't executed. (#2366)
-        [MethodImpl(MethodImplOptions.NoInlining)]
-        private unsafe ClockSpan MeasureWithRandomMemory(Action<long> action, long invokeCount)
+        [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)]
+        private unsafe ClockSpan MeasureWithRandomStack(Action<long> action, long invokeCount)
         {
             byte* stackMemory = stackalloc byte[random.Next(32)];
             var clockSpan = Measure(action, invokeCount);
@@ -205,6 +210,7 @@ private unsafe ClockSpan MeasureWithRandomMemory(Action<long> action, long invok
         [MethodImpl(MethodImplOptions.NoInlining)]
         private unsafe void Consume(byte* _) { }
 
+        [MethodImpl(MethodImplOptions.NoInlining | CodeGenHelper.AggressiveOptimizationOption)]
         private ClockSpan Measure(Action<long> action, long invokeCount)
         {
             var clock = Clock.Start();

src/BenchmarkDotNet/Engines/EngineActualStage.cs

@@ -0,0 +1,96 @@
+using System;
+using System.Collections.Generic;
+using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Mathematics;
+using BenchmarkDotNet.Reports;
+using Perfolizer.Horology;
+using Perfolizer.Mathematics.OutlierDetection;
+
+namespace BenchmarkDotNet.Engines
+{
+    internal abstract class EngineActualStage(IterationMode iterationMode) : EngineStage(IterationStage.Actual, iterationMode)
+    {
+        internal const int MaxOverheadIterationCount = 20;
+
+        internal static EngineActualStage GetOverhead(IEngine engine)
+            => new EngineActualStageAuto(engine.TargetJob, engine.Resolver, IterationMode.Overhead);
+
+        internal static EngineActualStage GetWorkload(IEngine engine, RunStrategy strategy)
+        {
+            var targetJob = engine.TargetJob;
+            int? iterationCount = targetJob.ResolveValueAsNullable(RunMode.IterationCountCharacteristic);
+            const int DefaultWorkloadCount = 10;
+            return iterationCount == null && strategy != RunStrategy.Monitoring
+                ?  new EngineActualStageAuto(targetJob, engine.Resolver, IterationMode.Workload)
+                :  new EngineActualStageSpecific(iterationCount ?? DefaultWorkloadCount, IterationMode.Workload);
+        }
+    }
+
+    internal sealed class EngineActualStageAuto : EngineActualStage
+    {
+        private readonly double maxRelativeError;
+        private readonly TimeInterval? maxAbsoluteError;
+        private readonly OutlierMode outlierMode;
+        private readonly int minIterationCount;
+        private readonly int maxIterationCount;
+        private readonly List<Measurement> measurementsForStatistics;
+        private int iterationCounter = 0;
+
+        public EngineActualStageAuto(Job targetJob, IResolver resolver, IterationMode iterationMode) : base(iterationMode)
+        {
+            maxRelativeError = targetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, resolver);
+            maxAbsoluteError = targetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic);
+            outlierMode = targetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, resolver);
+            minIterationCount = targetJob.ResolveValue(RunMode.MinIterationCountCharacteristic, resolver);
+            maxIterationCount = targetJob.ResolveValue(RunMode.MaxIterationCountCharacteristic, resolver);
+            measurementsForStatistics = GetMeasurementList();
+        }
+
+        internal override List<Measurement> GetMeasurementList() => new (maxIterationCount);
+
+        internal override bool GetShouldRunIteration(List<Measurement> measurements, ref long invokeCount)
+        {
+            if (measurements.Count == 0)
+            {
+                return true;
+            }
+
+            const double MaxOverheadRelativeError = 0.05;
+            bool isOverhead = Mode == IterationMode.Overhead;
+            double effectiveMaxRelativeError = isOverhead ? MaxOverheadRelativeError : maxRelativeError;
+            iterationCounter++;
+            var measurement = measurements[measurements.Count - 1];
+            measurementsForStatistics.Add(measurement);
+
+            var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, outlierMode);
+            double actualError = statistics.LegacyConfidenceInterval.Margin;
+
+            double maxError1 = effectiveMaxRelativeError * statistics.Mean;
+            double maxError2 = maxAbsoluteError?.Nanoseconds ?? double.MaxValue;
+            double maxError = Math.Min(maxError1, maxError2);
+
+            if (iterationCounter >= minIterationCount && actualError < maxError)
+            {
+                return false;
+            }
+
+            if (iterationCounter >= maxIterationCount || isOverhead && iterationCounter >= MaxOverheadIterationCount)
+            {
+                return false;
+            }
+
+            return true;
+        }
+    }
+
+    internal sealed class EngineActualStageSpecific(int maxIterationCount, IterationMode iterationMode) : EngineActualStage(iterationMode)
+    {
+        private int iterationCount = 0;
+
+        internal override List<Measurement> GetMeasurementList() => new (maxIterationCount);
+
+        internal override bool GetShouldRunIteration(List<Measurement> measurements, ref long invokeCount)
+            => ++iterationCount <= maxIterationCount;
+    }
+}