Skip to content

Commit afdb284

Browse files
MarcelKochMarcelKoch
committed
[bench] [bench] move conversion operations to json input
1 parent ccec40a commit afdb284

12 files changed

+412
-174
lines changed

‎benchmark/conversion/conversion.cpp‎

Lines changed: 69 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -33,31 +33,8 @@ using Generator = DefaultSystemGenerator<>;
3333

3434
struct ConversionBenchmark : Benchmark<gko::device_matrix_data<etype, itype>> {
3535
std::string name;
36-
std::vector<std::string> operations;
3736

38-
ConversionBenchmark() : name{"conversion"}
39-
{
40-
auto ref_exec = gko::ReferenceExecutor::create();
41-
auto formats = split(FLAGS_formats);
42-
for (const auto& from_format : formats) {
43-
operations.push_back(from_format + "-read");
44-
auto from_mtx =
45-
formats::matrix_type_factory.at(from_format)(ref_exec);
46-
// all pairs of conversions that are supported by Ginkgo
47-
for (const auto& to_format : formats) {
48-
if (from_format == to_format) {
49-
continue;
50-
}
51-
auto to_mtx =
52-
formats::matrix_type_factory.at(to_format)(ref_exec);
53-
try {
54-
to_mtx->copy_from(from_mtx);
55-
operations.push_back(from_format + "-" + to_format);
56-
} catch (const std::exception& e) {
57-
}
58-
}
59-
}
60-
}
37+
ConversionBenchmark() : name{"conversion"} {}
6138

6239
const std::string& get_name() const override { return name; }
6340

@@ -83,55 +60,80 @@ struct ConversionBenchmark : Benchmark<gko::device_matrix_data<etype, itype>> {
8360
gko::device_matrix_data<etype, itype>& data,
8461
const json& operation_case, json& result_case) const override
8562
{
86-
for (const auto& operation_name : operations) {
87-
result_case[operation_name] = json::object();
88-
auto& op_result_case = result_case[operation_name];
89-
90-
auto split_it =
91-
std::find(operation_name.begin(), operation_name.end(), '-');
92-
std::string from_name{operation_name.begin(), split_it};
93-
std::string to_name{split_it + 1, operation_name.end()};
94-
auto mtx_from = formats::matrix_type_factory.at(from_name)(exec);
95-
auto readable = gko::as<gko::ReadableFromMatrixData<etype, itype>>(
96-
mtx_from.get());
97-
IterationControl ic{timer};
98-
if (to_name == "read") {
99-
// warm run
100-
{
101-
auto range = annotate("warmup", FLAGS_warmup > 0);
102-
for (auto _ : ic.warmup_run()) {
103-
exec->synchronize();
104-
readable->read(data);
105-
exec->synchronize();
106-
}
107-
}
108-
// timed run
109-
for (auto _ : ic.run()) {
110-
auto range = annotate("repetition");
63+
std::string from_name = operation_case["from"].get<std::string>();
64+
std::string to_name = operation_case["to"].get<std::string>();
65+
auto mtx_from = formats::matrix_type_factory.at(from_name)(exec);
66+
auto readable =
67+
gko::as<gko::ReadableFromMatrixData<etype, itype>>(mtx_from.get());
68+
69+
// check if conversion is supported on empty matrix first
70+
if (from_name != to_name) {
71+
auto to_mtx = formats::matrix_type_factory.at(to_name)(exec);
72+
to_mtx->copy_from(mtx_from);
73+
}
74+
75+
IterationControl ic{timer};
76+
if (to_name == from_name) {
77+
// warm run
78+
{
79+
auto range = annotate("warmup", FLAGS_warmup > 0);
80+
for (auto _ : ic.warmup_run()) {
81+
exec->synchronize();
11182
readable->read(data);
83+
exec->synchronize();
11284
}
113-
} else {
85+
}
86+
// timed run
87+
for (auto _ : ic.run()) {
88+
auto range = annotate("repetition");
11489
readable->read(data);
115-
auto mtx_to = formats::matrix_type_factory.at(to_name)(exec);
116-
117-
// warm run
118-
{
119-
auto range = annotate("warmup", FLAGS_warmup > 0);
120-
for (auto _ : ic.warmup_run()) {
121-
exec->synchronize();
122-
mtx_to->copy_from(mtx_from);
123-
exec->synchronize();
124-
}
125-
}
126-
// timed run
127-
for (auto _ : ic.run()) {
128-
auto range = annotate("repetition");
90+
}
91+
} else {
92+
readable->read(data);
93+
auto mtx_to = formats::matrix_type_factory.at(to_name)(exec);
94+
95+
// warm run
96+
{
97+
auto range = annotate("warmup", FLAGS_warmup > 0);
98+
for (auto _ : ic.warmup_run()) {
99+
exec->synchronize();
129100
mtx_to->copy_from(mtx_from);
101+
exec->synchronize();
130102
}
131103
}
132-
op_result_case["time"] = ic.compute_time(FLAGS_timer_method);
133-
op_result_case["repetitions"] = ic.get_num_repetitions();
104+
// timed run
105+
for (auto _ : ic.run()) {
106+
auto range = annotate("repetition");
107+
mtx_to->copy_from(mtx_from);
108+
}
109+
}
110+
result_case["time"] = ic.compute_time(FLAGS_timer_method);
111+
result_case["repetitions"] = ic.get_num_repetitions();
112+
}
113+
114+
void postprocess(json& test_cases) const override
115+
{
116+
std::map<json, json> same_operators;
117+
for (const auto& test_case : test_cases) {
118+
if (test_case[name].contains("error_type") &&
119+
test_case[name]["error_type"] == "gko::NotSupported") {
120+
continue;
121+
}
122+
auto case_operator = test_case;
123+
case_operator.erase("to");
124+
case_operator.erase("from");
125+
case_operator.erase(name);
126+
same_operators.try_emplace(case_operator, json::array());
127+
same_operators[case_operator].push_back(test_case[name]);
128+
same_operators[case_operator].back()["to"] = test_case["to"];
129+
same_operators[case_operator].back()["from"] = test_case["from"];
134130
}
131+
auto merged_cases = json::array();
132+
for (auto& [case_operator, results] : same_operators) {
133+
merged_cases.push_back(case_operator);
134+
merged_cases.back()[name] = results;
135+
}
136+
test_cases = std::move(merged_cases);
135137
}
136138
};
137139

@@ -146,12 +148,8 @@ int main(int argc, char* argv[])
146148

147149
initialize_argument_parsing(&argc, &argv, header, schema["examples"]);
148150

149-
std::string extra_information =
150-
std::string() + "The formats are " + FLAGS_formats;
151-
152151
auto exec = executor_factory.at(FLAGS_executor)(FLAGS_gpu_timer);
153-
print_general_information(extra_information, exec);
154-
auto formats = split(FLAGS_formats, ',');
152+
print_general_information("", exec);
155153

156154
auto test_cases = json::parse(get_input_stream());
157155

‎benchmark/schema/conversion.json‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,14 @@
66
"properties": {
77
"operator": {
88
"$ref": "operator.json"
9+
},
10+
"to": {
11+
"type": "string",
12+
"default": "coo"
13+
},
14+
"from": {
15+
"type": "string",
16+
"default": "csr"
917
}
1018
},
1119
"required": [
@@ -29,7 +37,19 @@
2937
"name": "5pt",
3038
"size": 100
3139
}
32-
}
40+
},
41+
"to": "coo",
42+
"from": "coo"
43+
},
44+
{
45+
"operator": {
46+
"stencil": {
47+
"name": "5pt",
48+
"size": 100
49+
}
50+
},
51+
"to": "csr",
52+
"from": "coo"
3353
}
3454
]
3555
}

‎benchmark/test/conversion.py‎

Lines changed: 26 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,29 @@
11
#!/usr/bin/env python3
22
import test_framework
3+
from typing import List, Tuple
4+
import json
35

4-
stencil_input = '[{"operator": {"stencil": {"name": "7pt", "size": 100}}}]'
56

7+
def generate_input(operations: List[Tuple[str]]):
8+
input = [{"operator": {"stencil": {"name": "7pt", "size": 100}}, "to": to_, "from": from_} for from_, to_ in
9+
operations]
10+
return json.dumps(input)
11+
12+
13+
stencil_input = generate_input([("coo", "coo"), ("coo", "csr"), ("csr", "csr"), ("csr", "coo")])
14+
stencil_input_all = generate_input([("coo", "coo"),
15+
("coo", "csr"),
16+
("csr", "csr"),
17+
("csr", "coo"),
18+
("csr", "ell"),
19+
("csr", "sellp"),
20+
("csr", "hybrid"),
21+
("ell", "ell"),
22+
("ell", "csr"),
23+
("sellp", "sellp"),
24+
("sellp", "csr"),
25+
("hybrid", "hybrid"),
26+
("hybrid", "csr")])
627

728
# check that all input modes work:
829
# parameter
@@ -14,7 +35,7 @@
1435

1536
# stdin
1637
test_framework.compare_output(
17-
["-formats", "coo,csr"],
38+
[],
1839
expected_stdout="conversion.simple.stdout",
1940
expected_stderr="conversion.simple.stderr",
2041
stdin=stencil_input,
@@ -24,9 +45,7 @@
2445
test_framework.compare_output(
2546
[
2647
"-input",
27-
str(test_framework.sourcepath / "input.mtx.json"),
28-
"-formats",
29-
"coo,csr",
48+
str(test_framework.sourcepath / "input.conversion.json")
3049
],
3150
expected_stdout="conversion.simple.stdout",
3251
expected_stderr="conversion.simple.stderr",
@@ -36,9 +55,7 @@
3655
test_framework.compare_output(
3756
[
3857
"-input",
39-
stencil_input,
40-
"-formats",
41-
"coo,csr,ell,sellp,hybrid",
58+
stencil_input_all
4259
],
4360
expected_stdout="conversion.all.stdout",
4461
expected_stderr="conversion.all.stderr",
@@ -49,8 +66,6 @@
4966
[
5067
"-input",
5168
stencil_input,
52-
"-formats",
53-
"coo,csr",
5469
"-profile",
5570
"-profiler_hook",
5671
"debug",
@@ -61,7 +76,7 @@
6176

6277
# complex
6378
test_framework.compare_output(
64-
["-input", stencil_input, "-formats", "coo,csr"],
79+
["-input", stencil_input],
6580
expected_stdout="conversion_dcomplex.simple.stdout",
6681
expected_stderr="conversion_dcomplex.simple.stderr",
6782
use_complex=True

‎benchmark/test/input.conversion.json‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
[
2+
{
3+
"operator": {
4+
"stencil": {
5+
"name": "7pt",
6+
"size": 100
7+
}
8+
},
9+
"to": "coo",
10+
"from": "coo"
11+
},
12+
{
13+
"operator": {
14+
"stencil": {
15+
"name": "7pt",
16+
"size": 100
17+
}
18+
},
19+
"to": "csr",
20+
"from": "coo"
21+
},
22+
{
23+
"operator": {
24+
"stencil": {
25+
"name": "7pt",
26+
"size": 100
27+
}
28+
},
29+
"to": "csr",
30+
"from": "csr"
31+
},
32+
{
33+
"operator": {
34+
"stencil": {
35+
"name": "7pt",
36+
"size": 100
37+
}
38+
},
39+
"to": "coo",
40+
"from": "csr"
41+
}
42+
]

‎benchmark/test/reference/conversion.all.stderr‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,43 @@
11
Running on ReferenceExecutor
22
Running with 2 warm iterations and 10 running iterations
33
The random seed for right hand sides is 42
4-
The formats are coo,csr,ell,sellp,hybrid
4+
55
Running test case
6-
{"operator":{"stencil":{"name":"7pt","size":100}}}
6+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"coo","from":"coo"}
7+
Matrix is of size (125, 125), 725
8+
Running test case
9+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"csr","from":"coo"}
10+
Matrix is of size (125, 125), 725
11+
Running test case
12+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"csr","from":"csr"}
13+
Matrix is of size (125, 125), 725
14+
Running test case
15+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"coo","from":"csr"}
16+
Matrix is of size (125, 125), 725
17+
Running test case
18+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"ell","from":"csr"}
19+
Matrix is of size (125, 125), 725
20+
Running test case
21+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"sellp","from":"csr"}
22+
Matrix is of size (125, 125), 725
23+
Running test case
24+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"hybrid","from":"csr"}
25+
Matrix is of size (125, 125), 725
26+
Running test case
27+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"ell","from":"ell"}
28+
Matrix is of size (125, 125), 725
29+
Running test case
30+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"csr","from":"ell"}
31+
Matrix is of size (125, 125), 725
32+
Running test case
33+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"sellp","from":"sellp"}
34+
Matrix is of size (125, 125), 725
35+
Running test case
36+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"csr","from":"sellp"}
37+
Matrix is of size (125, 125), 725
38+
Running test case
39+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"hybrid","from":"hybrid"}
40+
Matrix is of size (125, 125), 725
41+
Running test case
42+
{"operator":{"stencil":{"name":"7pt","size":100}},"to":"csr","from":"hybrid"}
743
Matrix is of size (125, 125), 725

0 commit comments

Comments
 (0)