Skip to content

Commit b9d650b

Browse files
luffykailuffykai
committed
use new summarize script
1 parent a616bab commit b9d650b

File tree

2 files changed

+185
-5
lines changed

2 files changed

+185
-5
lines changed

‎.github/workflows/reth-benchmark.yml‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -173,11 +173,18 @@ jobs:
173173
echo "UPLOAD_FLAMEGRAPHS=1" >> $GITHUB_ENV
174174
fi
175175
176-
- name: Add benchmark metadata to markdown, and upload
176+
# - name: Add benchmark metadata to markdown, and upload
177+
# run: |
178+
# source afs-prototype/ci/scripts/utils.sh
179+
# add_metadata results.md ${{ inputs.max_segment_length }} ${{ inputs.instance_family }} ${{ inputs.memory_allocator }} ${{ github.repository }} ${{ github.run_id }}
180+
# s5cmd cp "${{ env.S3_PATH }}/${{ env.current_sha }}-${METRIC_NAME}.md" "${{ env.S3_PATH }}/main-${METRIC_NAME}.md"
181+
182+
- name: Summarize metrics
177183
run: |
178-
source afs-prototype/ci/scripts/utils.sh
179-
add_metadata results.md ${{ inputs.max_segment_length }} ${{ inputs.instance_family }} ${{ inputs.memory_allocator }} ${{ github.repository }} ${{ github.run_id }}
180-
s5cmd cp "${{ env.S3_PATH }}/${{ env.current_sha }}-${METRIC_NAME}.md" "${{ env.S3_PATH }}/main-${METRIC_NAME}.md"
184+
if [[ -f $METRIC_PATH ]]; then
185+
python3 ci/summarize.py $METRIC_PATH --print >> results.md
186+
s5cmd cp results.md "${{ env.S3_PATH }}/${current_sha}-${METRIC_NAME}.md"
187+
fi
181188
182189
### Update gh-pages
183190
- uses: actions/checkout@v4
@@ -194,7 +201,7 @@ jobs:
194201
GH_PAGES_PATH="benchmarks-dispatch/${{ github.head_ref || github.ref }}"
195202
echo "GH_PAGES_PATH=${GH_PAGES_PATH}" >> $GITHUB_ENV
196203
mkdir -p ${GH_PAGES_PATH}
197-
s5cmd cp "${{ env.S3_PATH }}/main-${METRIC_NAME}.md" "${GH_PAGES_PATH}/${METRIC_NAME}.md"
204+
s5cmd cp "${{ env.S3_PATH }}/${current_sha}-${METRIC_NAME}.md" "${GH_PAGES_PATH}/${METRIC_NAME}.md"
198205
199206
git add ${GH_PAGES_PATH}/${METRIC_NAME}.md
200207
git commit --allow-empty -m "Update benchmark result at ${GH_PAGES_PATH}/${METRIC_NAME}.md"

‎ci/summarize.py‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
import json
2+
import argparse
3+
import re
4+
from tabulate import tabulate
5+
6+
METRIC_TAGS = {
7+
'halo2_proof_time_ms': 'Halo2 Prove',
8+
'execute_time_ms': 'Execution',
9+
'stark_prove_excluding_trace_time_ms': 'STARK Prove',
10+
'trace_gen_time_ms': 'Tracegen',
11+
}
12+
13+
LABEL_PRIORITY = {
14+
'group': 0,
15+
'idx': 1,
16+
'segment': 2,
17+
'hgt': 3,
18+
'block_number': 4,
19+
}
20+
21+
LABEL_TAG = {
22+
'group': '',
23+
'idx': 'idx=',
24+
'segment': 'seg=',
25+
'hgt': 'hgt=',
26+
'block_number': 'blk=',
27+
}
28+
29+
def process_label(label):
30+
if 'height' in label:
31+
label = re.split('_height', label)[0]
32+
if label in ['leaf_verifier', 'internal_verifier', 'root_verifier']:
33+
label = re.split('_verifier', label)[0]
34+
return label
35+
36+
def main():
37+
argparser = argparse.ArgumentParser()
38+
argparser.add_argument('metrics_json', type=str, help="Path to the metrics JSON")
39+
argparser.add_argument('--out', type=str, help="Path to the output file")
40+
argparser.add_argument('--print', action='store_true', help="Print the output to the console", default=False)
41+
argparser.add_argument('--print-raw', action='store_true', help="Print the raw metrics to the console", default=False)
42+
args = argparser.parse_args()
43+
44+
with open(args.metrics_json, 'r') as f:
45+
x = json.load(f)
46+
47+
x = x['gauge'] # contains all the timing metrics, in ms
48+
x = [y for y in x if y['metric'] not in ['halo2_total_cells', 'halo2_keygen_time_ms']]
49+
for y in x:
50+
if 'group' not in [a[0] for a in y['labels']]:
51+
y['labels'].append(['group', 'dummy'])
52+
metrics = set([y['metric'] for y in x])
53+
54+
block_number = 0
55+
z = {}
56+
for m in metrics:
57+
tag = METRIC_TAGS[m]
58+
z[tag] = {}
59+
group_sum = 0
60+
for y in x:
61+
if y['metric'] == m:
62+
y_tags = []
63+
for xx in sorted(y['labels'], key=lambda a: LABEL_PRIORITY[a[0]]):
64+
if not (xx[0] in ['block_number']):
65+
y_tags.append(LABEL_TAG[xx[0]] + process_label(xx[1]))
66+
else:
67+
block_number = int(xx[1])
68+
y_str = '|'.join(y_tags)
69+
sec = float(y['value']) / 1000
70+
if 'agg_keygen' not in y_str:
71+
z[tag][y_str] = [sec, sec / 60]
72+
group_sum += sec
73+
z[tag]['Total'] = [group_sum, group_sum / 60]
74+
75+
parallel = {}
76+
for key in ['Execution', 'Tracegen', 'STARK Prove', 'Halo2 Prove']:
77+
parallel[key] = []
78+
79+
INTERNAL_GROUPS = ['internal_0', 'internal_1', 'internal_2', 'internal_3', 'internal_4', 'internal_5', 'internal_6', 'internal_7', 'internal_8', 'internal_9']
80+
81+
for grp in ['reth_block', 'dummy', 'leaf', 'internal', 'root'] + INTERNAL_GROUPS:
82+
if grp != 'internal':
83+
val = max([a['value'] for a in x if a['metric'] == 'execute_time_ms' and ['group', grp] in a['labels']], default=0)
84+
parallel['Execution'].append([grp, float(val) / 1000, float(val) / 60000])
85+
else:
86+
heights = set(list([[b[1] for b in a['labels'] if b[0] == 'hgt'][0] for a in x if a['metric'] == 'execute_time_ms' and ['group', grp] in a['labels']]))
87+
for hgt in heights:
88+
val = max([a['value'] for a in x if a['metric'] == 'execute_time_ms' and ['group', grp] in a['labels'] and ['hgt', hgt] in a['labels']], default=0)
89+
parallel['Execution'].append([grp + '_' + hgt, float(val) / 1000, float(val) / 60000])
90+
91+
for grp in ['reth_block', 'dummy', 'leaf', 'internal', 'root'] + INTERNAL_GROUPS:
92+
if grp != 'internal':
93+
val = max([a['value'] for a in x if a['metric'] == 'trace_gen_time_ms' and ['group', grp] in a['labels']], default=0)
94+
parallel['Tracegen'].append([grp, float(val) / 1000, float(val) / 60000])
95+
else:
96+
heights = set(list([[b[1] for b in a['labels'] if b[0] == 'hgt'][0] for a in x if a['metric'] == 'trace_gen_time_ms' and ['group', grp] in a['labels']]))
97+
for hgt in heights:
98+
val = max([a['value'] for a in x if a['metric'] == 'trace_gen_time_ms' and ['group', grp] in a['labels'] and ['hgt', hgt] in a['labels']], default=0)
99+
parallel['Tracegen'].append([grp + '_' + hgt, float(val) / 1000, float(val) / 60000])
100+
101+
for grp in ['reth_block', 'dummy', 'leaf', 'internal', 'root'] + INTERNAL_GROUPS:
102+
if grp != 'internal':
103+
val = max([a['value'] for a in x if a['metric'] == 'stark_prove_excluding_trace_time_ms' and ['group', grp] in a['labels']], default=0)
104+
parallel['STARK Prove'].append([grp, float(val) / 1000, float(val) / 60000])
105+
else:
106+
heights = set(list([[b[1] for b in a['labels'] if b[0] == 'hgt'][0] for a in x if a['metric'] == 'stark_prove_excluding_trace_time_ms' and ['group', grp] in a['labels']]))
107+
for hgt in heights:
108+
val = max([a['value'] for a in x if a['metric'] == 'stark_prove_excluding_trace_time_ms' and ['group', grp] in a['labels'] and ['hgt', hgt] in a['labels']], default=0)
109+
parallel['STARK Prove'].append([grp + '_' + hgt, float(val) / 1000, float(val) / 60000])
110+
111+
for grp in ['halo2_outer', 'halo2_wrapper']:
112+
val = max([a['value'] for a in x if a['metric'] == 'halo2_proof_time_ms' and ['group', grp] in a['labels']], default=0)
113+
parallel['Halo2 Prove'].append([grp, float(val) / 1000, float(val) / 60000])
114+
115+
for tag, tbl in parallel.items():
116+
tbl.append(['Total', sum([a[1] for a in tbl]), sum([a[2] for a in tbl])])
117+
118+
total_sum = sum([z[tag]['Total'][0] for tag in z])
119+
120+
total_table = [
121+
['Total', total_sum, total_sum / 60, sum([a[-1][1] for a in parallel.values()]), sum([a[-1][2] for a in parallel.values()])],
122+
['Execution', z['Execution']['Total'][0], z['Execution']['Total'][1], parallel['Execution'][-1][1], parallel['Execution'][-1][2]],
123+
['Tracegen', z['Tracegen']['Total'][0], z['Tracegen']['Total'][1], parallel['Tracegen'][-1][1], parallel['Tracegen'][-1][2]],
124+
['STARK Prove', z['STARK Prove']['Total'][0], z['STARK Prove']['Total'][1], parallel['STARK Prove'][-1][1], parallel['STARK Prove'][-1][2]],
125+
['Halo2 Prove', z['Halo2 Prove']['Total'][0], z['Halo2 Prove']['Total'][1], parallel['Halo2 Prove'][-1][1], parallel['Halo2 Prove'][-1][2]],
126+
]
127+
for key, value in z['Tracegen'].items():
128+
if key in z['STARK Prove']:
129+
z['STARK Prove'][key].extend(value)
130+
else:
131+
split = re.split('\|seg=0', key)
132+
if len(split) > 0:
133+
key = split[0] + split[1]
134+
else:
135+
key = split[0]
136+
if key in z['STARK Prove']:
137+
z['STARK Prove'][key].extend(value)
138+
else:
139+
print("ERROR: Key not found {}".format(key))
140+
141+
for key, value in z['STARK Prove'].items():
142+
if len(value) != 4:
143+
value.extend([0, 0])
144+
145+
exec_table = list(map(lambda a: [a[0], a[1][0], a[1][1]], z['Execution'].items()))
146+
stark_table = list(map(lambda a: [a[0], a[1][0], a[1][1], a[1][2], a[1][3]], z['STARK Prove'].items()))
147+
halo2_table = list(map(lambda a: [a[0], a[1][0], a[1][1]], z['Halo2 Prove'].items()))
148+
149+
if args.out:
150+
with open(args.out, 'w') as f:
151+
print(tabulate(total_table, headers=['Block ' + str(block_number), 'time (s)', 'time (m)', 'partime (s)', 'partime (m)'], tablefmt="pipe", floatfmt=".2f"), file=f)
152+
print(file=f)
153+
print(tabulate(exec_table, headers=['Block ' + str(block_number), 'Execution (s)', 'Execution (m)'], tablefmt="pipe", floatfmt=".2f"), file=f)
154+
print(file=f)
155+
print(tabulate(stark_table, headers=['Block ' + str(block_number), 'STARK Prove (s)', 'STARK Prove (m)', 'Tracegen (s)', 'Tracegen (m)'], tablefmt="pipe", floatfmt=".2f"), file=f)
156+
print(file=f)
157+
print(tabulate(halo2_table, headers=['Block ' + str(block_number), 'Halo2 Prove (s)', 'Halo2 Prove (m)'], tablefmt="pipe", floatfmt=".2f"), file=f)
158+
159+
if args.print:
160+
print(tabulate(total_table, headers=['Block ' + str(block_number), 'time (s)', 'time (m)', 'partime (s)', 'partime (m)'], tablefmt="pipe", floatfmt=".2f"))
161+
print()
162+
print(tabulate(exec_table, headers=['Block ' + str(block_number), 'Execution (s)', 'Execution (m)'], tablefmt="pipe", floatfmt=".2f"))
163+
print()
164+
print(tabulate(stark_table, headers=['Block ' + str(block_number), 'STARK Prove (s)', 'STARK Prove (m)', 'Tracegen (s)', 'Tracegen (m)'], tablefmt="pipe", floatfmt=".2f"))
165+
print()
166+
print(tabulate(halo2_table, headers=['Block ' + str(block_number), 'Halo2 Prove (s)', 'Halo2 Prove (m)'], tablefmt="pipe", floatfmt=".2f"))
167+
168+
if args.print_raw:
169+
for y in x:
170+
print(y)
171+
172+
if __name__ == '__main__':
173+
main()

0 commit comments

Comments
 (0)