[GR-64081] Implement new 'graalos' bench suite

PullRequest: graal/20779

Author: Andrija Kolic

common.json

@@ -4,7 +4,7 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "7.51.2",
+  "mx_version": "7.54.3",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {

sdk/mx.sdk/mx_sdk_benchmark.py

@@ -71,7 +71,7 @@
 import mx_sdk_vm
 import mx_sdk_vm_impl
 import mx_util
-from mx_benchmark import DataPoints, DataPoint, BenchmarkSuite
+from mx_benchmark import DataPoints, DataPoint, BenchmarkSuite, Vm, SingleBenchmarkExecutionContext
 from mx_sdk_vm_impl import svm_experimental_options
 
 _suite = mx.suite('sdk')
@@ -2815,17 +2815,8 @@ def __init__(self, custom_harness_command: mx_benchmark.CustomHarnessCommand = N
             custom_harness_command = BaristaBenchmarkSuite.BaristaCommand()
         super().__init__(custom_harness_command)
         self._version = None
-        self._context = None
         self._extra_run_options = []
 
-    @property
-    def context(self):
-        return self._context
-
-    @context.setter
-    def context(self, value):
-        self._context = value
-
     def readBaristaVersionFromPyproject(self):
         # tomllib was included in python standard library with version 3.11
         try:
@@ -2858,6 +2849,9 @@ def group(self):
     def subgroup(self):
         return "graal-compiler"
 
+    def benchmarkName(self):
+        return self.execution_context.benchmark
+
     def benchmarkList(self, bmSuiteArgs):
         exclude = []
         # Barista currently does not support running 'micronaut-pegasus' on the JVM - running it results in a crash (GR-59793)
@@ -2892,15 +2886,18 @@ def baristaHarnessPath(self):
         return self.baristaFilePath("barista")
 
     def baristaHarnessBenchmarkName(self):
-        return _baristaConfig["benchmarks"][self.context.benchmark].get("barista-bench-name", self.context.benchmark)
+        return _baristaConfig["benchmarks"][self.benchmarkName()].get("barista-bench-name", self.benchmarkName())
 
     def baristaHarnessBenchmarkWorkload(self):
-        return _baristaConfig["benchmarks"][self.context.benchmark].get("workload")
+        return _baristaConfig["benchmarks"][self.benchmarkName()].get("workload")
 
     def validateEnvironment(self):
         self.baristaProjectConfigurationPath()
         self.baristaHarnessPath()
 
+    def new_execution_context(self, vm: Vm, benchmarks: List[str], bmSuiteArgs: List[str]) -> SingleBenchmarkExecutionContext:
+        return SingleBenchmarkExecutionContext(self, vm, benchmarks, bmSuiteArgs)
+
     def register_tracker(self, name, tracker_type):
         if tracker_type in _baristaConfig["disable_trackers"]:
             mx.log(f"Ignoring the registration of '{name}' tracker as it was disabled for {self.__class__.__name__}.")
@@ -2934,7 +2931,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):
 
         # Startup
         all_rules.append(mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": self.context.benchmark,
+            "benchmark": self.benchmarkName(),
             "metric.name": "request-time",
             "metric.type": "numeric",
             "metric.unit": "ms",
@@ -2947,7 +2944,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):
 
         # Warmup
         all_rules.append(mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": self.context.benchmark,
+            "benchmark": self.benchmarkName(),
             "metric.name": "warmup",
             "metric.type": "numeric",
             "metric.unit": "op/s",
@@ -2960,7 +2957,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):
 
         # Throughput
         all_rules.append(mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": self.context.benchmark,
+            "benchmark": self.benchmarkName(),
             "metric.name": "throughput",
             "metric.type": "numeric",
             "metric.unit": "op/s",
@@ -2973,7 +2970,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):
 
         # Latency
         all_rules += [mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": self.context.benchmark,
+            "benchmark": self.benchmarkName(),
             "metric.name": "latency",
             "metric.type": "numeric",
             "metric.unit": "ms",
@@ -2992,7 +2989,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):
 
         # Resource Usage
         all_rules += [mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": self.context.benchmark,
+            "benchmark": self.benchmarkName(),
             "metric.name": "rss",
             "metric.type": "numeric",
             "metric.unit": "MB",
@@ -3082,17 +3079,6 @@ def extendDatapoints(self, datapoints: DataPoints) -> DataPoints:
                 del datapoint["load-tester.command"]
         return datapoints
 
-    def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs):
-        self.enforce_single_benchmark(benchmarks)
-        self.context = BaristaBenchmarkSuite.RuntimeContext(self, vm, benchmarks[0], bmSuiteArgs)
-        return super()._vmRun(vm, workdir, command, benchmarks, bmSuiteArgs)
-
-    def enforce_single_benchmark(self, benchmarks):
-        if not isinstance(benchmarks, list):
-            raise TypeError(f"{self.__class__.__name__} expects to receive a list of benchmarks to run, instead got an instance of {benchmarks.__class__.__name__}! Please specify a single benchmark!")
-        if len(benchmarks) != 1:
-            raise ValueError(f"You have requested {benchmarks} to be run but {self.__class__.__name__} can only run a single benchmark at a time! Please specify a single benchmark!")
-
     class BaristaCommand(mx_benchmark.CustomHarnessCommand):
         """Maps a JVM command into a command tailored for the Barista harness.
         """
@@ -3151,7 +3137,7 @@ def produceHarnessCommand(self, cmd, suite):
             jvm_vm_options = jvm_cmd[index_of_java_exe + 1:]
 
             # Verify that the run arguments don't already contain a "--mode" option
-            run_args = suite.runArgs(suite.context.bmSuiteArgs) + suite._extra_run_options
+            run_args = suite.runArgs(suite.execution_context.bmSuiteArgs) + suite._extra_run_options
             mode_pattern = r"^(?:-m|--mode)(=.*)?$"
             mode_match = self._regexFindInCommand(run_args, mode_pattern)
             if mode_match:
@@ -3175,37 +3161,6 @@ def produceHarnessCommand(self, cmd, suite):
             barista_cmd.append(barista_bench_name)
             return barista_cmd
 
-    class RuntimeContext():
-        """Container class for the runtime context of BaristaBenchmarkSuite.
-        """
-        def __init__(self, suite, vm, benchmark, bmSuiteArgs):
-            if not isinstance(suite, BaristaBenchmarkSuite):
-                raise TypeError(f"Expected an instance of {BaristaBenchmarkSuite.__name__}, instead got an instance of {suite.__class__.__name__}")
-            self._suite = suite
-            self._vm = vm
-            self._benchmark = benchmark
-            self._bmSuiteArgs = bmSuiteArgs
-
-        @property
-        def suite(self):
-            return self._suite
-
-        @property
-        def vm(self):
-            return self._vm
-
-        @property
-        def benchmark(self):
-            """The currently running benchmark.
-
-            Corresponds to `benchmarks[0]` in a suite method that has a `benchmarks` argument.
-            """
-            return self._benchmark
-
-        @property
-        def bmSuiteArgs(self):
-            return self._bmSuiteArgs
-
 
 mx_benchmark.add_bm_suite(BaristaBenchmarkSuite())
 
@@ -3959,24 +3914,25 @@ def run(self, benchmarks, bm_suite_args: List[str]) -> DataPoints:
         fallback_reason = self.fallback_mode_reason(bm_suite_args)
 
         vm = self.get_vm_registry().get_vm_from_suite_args(bm_suite_args)
-        effective_stages, complete_stage_list = vm.prepare_stages(self, bm_suite_args)
-        self.stages_info = StagesInfo(effective_stages, complete_stage_list, vm, bool(fallback_reason))
-
-        if self.stages_info.fallback_mode:
-            # In fallback mode, all stages are run at once. There is matching code in `NativeImageVM.run_java` for this.
-            mx.log(f"Running benchmark in fallback mode (reason: {fallback_reason})")
-            datapoints += super_delegate.run(benchmarks, bm_suite_args)
-        else:
-            while self.stages_info.has_next_stage():
-                stage = self.stages_info.next_stage()
-                # Start the actual benchmark execution. The stages_info attribute will be used by the NativeImageVM to
-                # determine which stage to run this time.
-                stage_dps = super_delegate.run(benchmarks, bm_suite_args)
-                NativeImageBenchmarkMixin._inject_stage_keys(stage_dps, stage)
-                datapoints += stage_dps
-
-        self.stages_info = None
-        return datapoints
+        with self.new_execution_context(vm, benchmarks, bm_suite_args):
+            effective_stages, complete_stage_list = vm.prepare_stages(self, bm_suite_args)
+            self.stages_info = StagesInfo(effective_stages, complete_stage_list, vm, bool(fallback_reason))
+
+            if self.stages_info.fallback_mode:
+                # In fallback mode, all stages are run at once. There is matching code in `NativeImageVM.run_java` for this.
+                mx.log(f"Running benchmark in fallback mode (reason: {fallback_reason})")
+                datapoints += super_delegate.run(benchmarks, bm_suite_args)
+            else:
+                while self.stages_info.has_next_stage():
+                    stage = self.stages_info.next_stage()
+                    # Start the actual benchmark execution. The stages_info attribute will be used by the NativeImageVM to
+                    # determine which stage to run this time.
+                    stage_dps = super_delegate.run(benchmarks, bm_suite_args)
+                    NativeImageBenchmarkMixin._inject_stage_keys(stage_dps, stage)
+                    datapoints += stage_dps
+
+            self.stages_info = None
+            return datapoints
 
     @staticmethod
     def _inject_stage_keys(dps: DataPoints, stage: Stage) -> None: