[bench] adapt benchmarks to new input

Author: MarcelKoch

benchmark/blas/blas.cpp

@@ -106,17 +106,20 @@ Parameters for a benchmark case are:
     stride_B: stride for B matrix in gemm (optional, default m)
     stride_C: stride for C matrix in gemm (optional, default m)
 )";
-    std::string format = Generator::get_example_config();
-    initialize_argument_parsing(&argc, &argv, header, format);
+    auto schema =
+        json::parse(std::ifstream(GKO_ROOT "/benchmark/schema/blas.json"));
+
+    initialize_argument_parsing(&argc, &argv, header, schema["examples"]);
 
     std::string extra_information = "The operations are " + FLAGS_operations;
     auto exec = executor_factory.at(FLAGS_executor)(FLAGS_gpu_timer);
     print_general_information(extra_information, exec);
 
     auto test_cases = json::parse(get_input_stream());
 
-    run_test_cases(BlasBenchmark{operation_map}, exec,
-                   get_timer(exec, FLAGS_gpu_timer), test_cases);
+    auto results =
+        run_test_cases(BlasBenchmark{operation_map}, exec,
+                       get_timer(exec, FLAGS_gpu_timer), schema, test_cases);
 
-    std::cout << std::setw(4) << test_cases << std::endl;
+    std::cout << std::setw(4) << results << std::endl;
 }

benchmark/blas/blas_common.hpp

@@ -24,7 +24,7 @@
 // Command-line arguments
 DEFINE_string(
     operations, "copy,axpy,scal",
-    "A comma-separated list of operations to benchmark.\nCandidates are"
+    "A comma-separated list of operations to benchmark.\nCandidates are\n"
     "BLAS algorithms:\n"
     "   copy (y = x),\n"
     "   axpy (y = y + a * x),\n"
@@ -437,56 +437,18 @@ struct BlasBenchmark : Benchmark<dimensions> {
 
     const std::string& get_name() const override { return name; }
 
-    const std::vector<std::string>& get_operations() const override
-    {
-        return operations;
-    }
 
     bool should_print() const override { return do_print; }
 
-    std::string get_example_config() const override
-    {
-        return json::parse(R"([{"n": 100}, {"n": 200, "m": 200, "k": 200}])")
-            .dump(4);
-    }
-
-    bool validate_config(const json& value) const override
-    {
-        return value.contains("n") && value["n"].is_number_integer();
-    }
-
-    std::string describe_config(const json& test_case) const override
-    {
-        std::stringstream ss;
-        auto optional_output = [&](const char* name) {
-            if (test_case.contains(name) &&
-                test_case[name].is_number_integer()) {
-                ss << name << " = " << test_case[name].get<gko::int64>() << " ";
-            }
-        };
-        optional_output("n");
-        optional_output("k");
-        optional_output("m");
-        optional_output("r");
-        optional_output("stride");
-        optional_output("stride_x");
-        optional_output("stride_y");
-        optional_output("stride_A");
-        optional_output("stride_B");
-        optional_output("stride_C");
-        return ss.str();
-    }
-
     dimensions setup(std::shared_ptr<gko::Executor> exec,
                      json& test_case) const override
     {
         auto get_optional = [](json& obj, const char* name,
                                gko::size_type default_value) -> gko::size_type {
-            if (obj.contains(name)) {
-                return obj[name].get<gko::uint64>();
-            } else {
-                return default_value;
+            if (!obj.contains(name)) {
+                obj[name] = default_value;
             }
+            return obj[name].get<gko::uint64>();
         };
 
         dimensions result;
@@ -507,40 +469,43 @@ struct BlasBenchmark : Benchmark<dimensions> {
         return result;
     }
 
-
     void run(std::shared_ptr<gko::Executor> exec, std::shared_ptr<Timer> timer,
              annotate_functor annotate, dimensions& dims,
-             const std::string& operation_name,
-             json& operation_case) const override
+             const json& operation_case, json& result_case) const override
     {
-        auto op = operation_map.at(operation_name)(exec, dims);
+        for (auto& operation_name : operations) {
+            result_case[operation_name] = json::object();
+            auto& op_result_case = result_case[operation_name];
 
-        IterationControl ic(timer);
+            auto op = operation_map.at(operation_name)(exec, dims);
 
-        // warm run
-        {
-            auto range = annotate("warmup", FLAGS_warmup > 0);
-            for (auto _ : ic.warmup_run()) {
-                op->prepare();
-                exec->synchronize();
-                op->run();
-                exec->synchronize();
+            IterationControl ic(timer);
+
+            // warm run
+            {
+                auto range = annotate("warmup", FLAGS_warmup > 0);
+                for (auto _ : ic.warmup_run()) {
+                    op->prepare();
+                    exec->synchronize();
+                    op->run();
+                    exec->synchronize();
+                }
             }
-        }
 
-        // timed run
-        op->prepare();
-        for (auto _ : ic.run()) {
-            auto range = annotate("repetition");
-            op->run();
+            // timed run
+            op->prepare();
+            for (auto _ : ic.run()) {
+                auto range = annotate("repetition");
+                op->run();
+            }
+            const auto runtime = ic.compute_time(FLAGS_timer_method);
+            const auto flops = static_cast<double>(op->get_flops());
+            const auto mem = static_cast<double>(op->get_memory());
+            const auto repetitions = ic.get_num_repetitions();
+            op_result_case["time"] = runtime;
+            op_result_case["flops"] = flops / runtime;
+            op_result_case["bandwidth"] = mem / runtime;
+            op_result_case["repetitions"] = repetitions;
         }
-        const auto runtime = ic.compute_time(FLAGS_timer_method);
-        const auto flops = static_cast<double>(op->get_flops());
-        const auto mem = static_cast<double>(op->get_memory());
-        const auto repetitions = ic.get_num_repetitions();
-        operation_case["time"] = runtime;
-        operation_case["flops"] = flops / runtime;
-        operation_case["bandwidth"] = mem / runtime;
-        operation_case["repetitions"] = repetitions;
     }
 };

benchmark/blas/distributed/multi_vector.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors
+// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors
 //
 // SPDX-License-Identifier: BSD-3-Clause
 
@@ -38,8 +38,11 @@ Parameters for a benchmark case are:
     stride_x: stride for input vector x (optional, default r)
     stride_y: stride for in/out vector y (optional, default r)
 )";
-    std::string format = Generator::get_example_config();
-    initialize_argument_parsing(&argc, &argv, header, format, do_print);
+    auto schema = json::parse(
+        std::ifstream(GKO_ROOT "/benchmark/schema/blas-distributed.json"));
+
+    initialize_argument_parsing(&argc, &argv, header, schema["examples"],
+                                do_print);
 
     auto exec = executor_factory_mpi.at(FLAGS_executor)(comm.get());
 
@@ -98,10 +101,11 @@ Parameters for a benchmark case are:
                      exec, Generator{comm, {}}, dims.n, dims.r, dims.stride_y);
              }}};
 
-    run_test_cases(BlasBenchmark{operation_map, do_print}, exec,
-                   get_mpi_timer(exec, comm, FLAGS_gpu_timer), test_cases);
+    auto results = run_test_cases(BlasBenchmark{operation_map, do_print}, exec,
+                                  get_mpi_timer(exec, comm, FLAGS_gpu_timer),
+                                  schema, test_cases);
 
     if (do_print) {
-        std::cout << std::setw(4) << test_cases << std::endl;
+        std::cout << std::setw(4) << results << std::endl;
     }
 }

benchmark/conversion/conversion.cpp

@@ -61,28 +61,8 @@ struct ConversionBenchmark : Benchmark<gko::device_matrix_data<etype, itype>> {
 
     const std::string& get_name() const override { return name; }
 
-    const std::vector<std::string>& get_operations() const override
-    {
-        return operations;
-    }
-
     bool should_print() const override { return true; }
 
-    std::string get_example_config() const override
-    {
-        return Generator::get_example_config();
-    }
-
-    bool validate_config(const json& test_case) const override
-    {
-        return Generator::validate_config(test_case);
-    }
-
-    std::string describe_config(const json& test_case) const override
-    {
-        return Generator::describe_config(test_case);
-    }
-
     gko::device_matrix_data<etype, itype> setup(
         std::shared_ptr<gko::Executor> exec, json& test_case) const override
     {
@@ -98,57 +78,60 @@ struct ConversionBenchmark : Benchmark<gko::device_matrix_data<etype, itype>> {
                                                                        data);
     }
 
-
     void run(std::shared_ptr<gko::Executor> exec, std::shared_ptr<Timer> timer,
              annotate_functor annotate,
              gko::device_matrix_data<etype, itype>& data,
-             const std::string& operation_name,
-             json& operation_case) const override
+             const json& operation_case, json& result_case) const override
     {
-        auto split_it =
-            std::find(operation_name.begin(), operation_name.end(), '-');
-        std::string from_name{operation_name.begin(), split_it};
-        std::string to_name{split_it + 1, operation_name.end()};
-        auto mtx_from = formats::matrix_type_factory.at(from_name)(exec);
-        auto readable =
-            gko::as<gko::ReadableFromMatrixData<etype, itype>>(mtx_from.get());
-        IterationControl ic{timer};
-        if (to_name == "read") {
-            // warm run
-            {
-                auto range = annotate("warmup", FLAGS_warmup > 0);
-                for (auto _ : ic.warmup_run()) {
-                    exec->synchronize();
+        for (const auto& operation_name : operations) {
+            result_case[operation_name] = json::object();
+            auto& op_result_case = result_case[operation_name];
+
+            auto split_it =
+                std::find(operation_name.begin(), operation_name.end(), '-');
+            std::string from_name{operation_name.begin(), split_it};
+            std::string to_name{split_it + 1, operation_name.end()};
+            auto mtx_from = formats::matrix_type_factory.at(from_name)(exec);
+            auto readable = gko::as<gko::ReadableFromMatrixData<etype, itype>>(
+                mtx_from.get());
+            IterationControl ic{timer};
+            if (to_name == "read") {
+                // warm run
+                {
+                    auto range = annotate("warmup", FLAGS_warmup > 0);
+                    for (auto _ : ic.warmup_run()) {
+                        exec->synchronize();
+                        readable->read(data);
+                        exec->synchronize();
+                    }
+                }
+                // timed run
+                for (auto _ : ic.run()) {
+                    auto range = annotate("repetition");
                     readable->read(data);
-                    exec->synchronize();
                 }
-            }
-            // timed run
-            for (auto _ : ic.run()) {
-                auto range = annotate("repetition");
+            } else {
                 readable->read(data);
-            }
-        } else {
-            readable->read(data);
-            auto mtx_to = formats::matrix_type_factory.at(to_name)(exec);
-
-            // warm run
-            {
-                auto range = annotate("warmup", FLAGS_warmup > 0);
-                for (auto _ : ic.warmup_run()) {
-                    exec->synchronize();
+                auto mtx_to = formats::matrix_type_factory.at(to_name)(exec);
+
+                // warm run
+                {
+                    auto range = annotate("warmup", FLAGS_warmup > 0);
+                    for (auto _ : ic.warmup_run()) {
+                        exec->synchronize();
+                        mtx_to->copy_from(mtx_from);
+                        exec->synchronize();
+                    }
+                }
+                // timed run
+                for (auto _ : ic.run()) {
+                    auto range = annotate("repetition");
                     mtx_to->copy_from(mtx_from);
-                    exec->synchronize();
                 }
             }
-            // timed run
-            for (auto _ : ic.run()) {
-                auto range = annotate("repetition");
-                mtx_to->copy_from(mtx_from);
-            }
+            op_result_case["time"] = ic.compute_time(FLAGS_timer_method);
+            op_result_case["repetitions"] = ic.get_num_repetitions();
         }
-        operation_case["time"] = ic.compute_time(FLAGS_timer_method);
-        operation_case["repetitions"] = ic.get_num_repetitions();
     }
 };
 
@@ -157,8 +140,11 @@ int main(int argc, char* argv[])
 {
     std::string header =
         "A benchmark for measuring performance of Ginkgo's conversions.\n";
-    std::string format_str = Generator::get_example_config();
-    initialize_argument_parsing_matrix(&argc, &argv, header, format_str);
+
+    auto schema = json::parse(
+        std::ifstream(GKO_ROOT "/benchmark/schema/conversion.json"));
+
+    initialize_argument_parsing(&argc, &argv, header, schema["examples"]);
 
     std::string extra_information =
         std::string() + "The formats are " + FLAGS_formats;
@@ -169,8 +155,9 @@ int main(int argc, char* argv[])
 
     auto test_cases = json::parse(get_input_stream());
 
-    run_test_cases(ConversionBenchmark{}, exec,
-                   get_timer(exec, FLAGS_gpu_timer), test_cases);
+    auto results =
+        run_test_cases(ConversionBenchmark{}, exec,
+                       get_timer(exec, FLAGS_gpu_timer), schema, test_cases);
 
-    std::cout << std::setw(4) << test_cases << std::endl;
+    std::cout << std::setw(4) << results << std::endl;
 }