Skip to content

Commit d113408

Browse files
michaelstaibmichaelstaib
authored
Updated Benchmarking Flow (#8860)
1 parent 4445ea9 commit d113408

File tree

3 files changed

+84
-49
lines changed

3 files changed

+84
-49
lines changed

‎.github/workflows/benchmarks.yml‎

Lines changed: 51 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -6,29 +6,39 @@ on:
66
branches:
77
- main
88
- main-version-*
9-
paths-ignore:
10-
- "src/HotChocolate/AspNetCore/benchmarks/k6/performance-data.json"
9+
push:
10+
branches:
11+
- main
1112

1213
concurrency:
13-
group: benchmarks-${{ github.event.pull_request.number }}
14+
group: benchmarks-${{ github.event.pull_request.number || github.ref }}
1415
cancel-in-progress: true
1516

1617
jobs:
1718
hotchocolate-core:
1819
name: "HotChocolate Core Benchmarks"
19-
if: github.event.pull_request.draft == false
20+
if: github.event_name == 'push' || github.event.pull_request.draft == false
2021
runs-on: benchmarking
2122
permissions:
2223
contents: write
2324
pull-requests: write
2425

2526
steps:
26-
- name: Checkout PR code
27+
- name: Checkout current repository
2728
uses: actions/checkout@v4
2829
with:
2930
fetch-depth: 0
3031
show-progress: false
3132

33+
- name: Checkout performance data repository
34+
uses: actions/checkout@v4
35+
with:
36+
repository: ChilliCream/graphql-platform-performance-data
37+
token: ${{ secrets.PERFORMANCE_DATA_TOKEN }}
38+
path: performance-data-repo
39+
fetch-depth: 0
40+
show-progress: false
41+
3242
- name: Restore dependencies
3343
run: dotnet restore src/HotChocolate/AspNetCore/benchmarks/k6/eShop.slnx
3444

@@ -41,22 +51,22 @@ jobs:
4151
echo "APPHOST_PID=$APPHOST_PID" >> $GITHUB_ENV
4252
4353
echo "Waiting for server to be ready..."
44-
for i in {1..30}; do
54+
for i in {1..15}; do
4555
if curl -s -o /dev/null -w "%{http_code}" http://localhost:5224/graphql -X POST \
4656
-H "Content-Type: application/json" \
4757
-d '{"query": "{ __typename }"}' | grep -q "200"; then
4858
echo "Server is ready!"
4959
break
5060
fi
51-
echo "Waiting... ($i/30)"
61+
echo "Waiting... ($i/15)"
5262
sleep 2
5363
done
5464
5565
- name: Run performance tests and collect data
5666
working-directory: src/HotChocolate/AspNetCore/benchmarks/k6
5767
run: |
5868
chmod +x run-and-collect.sh
59-
./run-and-collect.sh performance-data-current.json
69+
./run-and-collect.sh hotchocolate-core-performance-data-current.json
6070
6171
- name: Stop AppHost
6272
if: always()
@@ -66,46 +76,28 @@ jobs:
6676
wait $APPHOST_PID 2>/dev/null || true
6777
fi
6878
69-
- name: Commit and push performance data to current branch
70-
working-directory: src/HotChocolate/AspNetCore/benchmarks/k6
71-
run: |
72-
# Copy the performance data to the tracked filename
73-
cp performance-data-current.json performance-data.json
74-
75-
# Configure git
76-
git config user.name "github-actions[bot]"
77-
git config user.email "github-actions[bot]@users.noreply.github.com"
78-
79-
# Add and commit the performance data
80-
git add performance-data.json
81-
82-
# Only commit if there are changes
83-
if ! git diff --staged --quiet; then
84-
git commit -m "Update performance data [skip ci]"
85-
git push origin HEAD:${{ github.head_ref }}
86-
else
87-
echo "No changes to performance data"
88-
fi
89-
90-
- name: Fetch baseline performance data from main
79+
- name: Fetch baseline performance data from external repo
80+
if: github.event_name == 'pull_request'
9181
run: |
92-
git fetch origin main:main
93-
if git show main:src/HotChocolate/AspNetCore/benchmarks/k6/performance-data.json > baseline-performance.json 2>/dev/null; then
94-
echo "Baseline data fetched successfully"
82+
if [ -f performance-data-repo/hotchocolate-core-performance-data.json ]; then
83+
echo "Baseline data fetched successfully from performance data repository"
84+
cp performance-data-repo/hotchocolate-core-performance-data.json baseline-performance.json
9585
cat baseline-performance.json
9686
else
97-
echo "No baseline data found on main branch"
87+
echo "No baseline data found in performance data repository"
9888
# Don't create the file - let the comparison script handle missing baseline
9989
rm -f baseline-performance.json
10090
fi
10191
10292
- name: Compare performance and generate report
93+
if: github.event_name == 'pull_request'
10394
working-directory: src/HotChocolate/AspNetCore/benchmarks/k6
10495
run: |
10596
chmod +x compare-performance.sh
106-
./compare-performance.sh performance-data-current.json ../../../../../baseline-performance.json performance-report.md
97+
./compare-performance.sh hotchocolate-core-performance-data-current.json ../../../../../baseline-performance.json performance-report.md
10798
10899
- name: Comment PR with performance report
100+
if: github.event_name == 'pull_request'
109101
uses: actions/github-script@v7
110102
with:
111103
github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -142,12 +134,13 @@ jobs:
142134
with:
143135
name: performance-data
144136
path: |
145-
src/HotChocolate/AspNetCore/benchmarks/k6/performance-data-current.json
137+
src/HotChocolate/AspNetCore/benchmarks/k6/hotchocolate-core-performance-data-current.json
146138
src/HotChocolate/AspNetCore/benchmarks/k6/performance-report.md
147139
/tmp/apphost.log
148140
retention-days: 30
149141

150142
- name: Check for performance regression
143+
if: github.event_name == 'pull_request'
151144
working-directory: src/HotChocolate/AspNetCore/benchmarks/k6
152145
run: |
153146
# Fail the build if there's a significant performance regression
@@ -156,3 +149,25 @@ jobs:
156149
# Uncomment the next line to fail the build on regression
157150
# exit 1
158151
fi
152+
153+
- name: Store performance data to external repository
154+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
155+
working-directory: performance-data-repo
156+
run: |
157+
# Copy the new performance data
158+
cp ../src/HotChocolate/AspNetCore/benchmarks/k6/hotchocolate-core-performance-data-current.json hotchocolate-core-performance-data.json
159+
160+
# Configure git
161+
git config user.name "github-actions[bot]"
162+
git config user.email "github-actions[bot]@users.noreply.github.com"
163+
164+
# Add and commit the performance data
165+
git add hotchocolate-core-performance-data.json
166+
167+
# Only commit if there are changes
168+
if ! git diff --staged --quiet; then
169+
git commit -m "Update HotChocolate core performance data from ${{ github.sha }}"
170+
git push
171+
else
172+
echo "No changes to performance data"
173+
fi

‎src/HotChocolate/AspNetCore/benchmarks/k6/compare-performance.sh‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -94,16 +94,30 @@ get_emoji() {
9494
# Function to format change
9595
format_change() {
9696
local change=$1
97-
local show_sign=${2:-true}
97+
local metric_type=$2 # "latency" or "throughput"
98+
local show_sign=${3:-true}
9899

99100
if [ "$change" == "0" ] || [ -z "$change" ]; then
100101
echo "(no change)"
101102
elif [ "$show_sign" == "true" ]; then
102-
if (( $(echo "$change > 0" | bc -l) )); then
103-
echo "(${change}% worse)"
103+
local is_positive=$(echo "$change > 0" | bc -l)
104+
local abs_change=$(echo "$change" | tr -d '-')
105+
106+
# For latency: lower is better (negative change = better)
107+
# For throughput: higher is better (positive change = better)
108+
if [ "$metric_type" == "throughput" ]; then
109+
if [ "$is_positive" == "1" ]; then
110+
echo "(${change}% better)"
111+
else
112+
echo "(${abs_change}% worse)"
113+
fi
104114
else
105-
local abs_change=$(echo "$change" | tr -d '-')
106-
echo "(${abs_change}% better)"
115+
# Default to latency behavior (lower is better)
116+
if [ "$is_positive" == "1" ]; then
117+
echo "(${change}% worse)"
118+
else
119+
echo "(${abs_change}% better)"
120+
fi
107121
fi
108122
else
109123
echo "(${change}%)"
@@ -238,15 +252,15 @@ if [ -n "$BASELINE_FILE" ]; then
238252
239253
| Test | Min | Med | Max | Avg | P90 | P95 | P99 |
240254
|------|-----|-----|-----|-----|-----|-----|-----|
241-
| **Single Fetch** | $(get_emoji "$CHANGE_SF_MIN" "latency") $(format_change "$CHANGE_SF_MIN") | $(get_emoji "$CHANGE_SF_P50" "latency") $(format_change "$CHANGE_SF_P50") | $(get_emoji "$CHANGE_SF_MAX" "latency") $(format_change "$CHANGE_SF_MAX") | $(get_emoji "$CHANGE_SF_AVG" "latency") $(format_change "$CHANGE_SF_AVG") | $(get_emoji "$CHANGE_SF_P90" "latency") $(format_change "$CHANGE_SF_P90") | $(get_emoji "$CHANGE_SF_P95" "latency") $(format_change "$CHANGE_SF_P95") | $(get_emoji "$CHANGE_SF_P99" "latency") $(format_change "$CHANGE_SF_P99") |
242-
| **DataLoader** | $(get_emoji "$CHANGE_DL_MIN" "latency") $(format_change "$CHANGE_DL_MIN") | $(get_emoji "$CHANGE_DL_P50" "latency") $(format_change "$CHANGE_DL_P50") | $(get_emoji "$CHANGE_DL_MAX" "latency") $(format_change "$CHANGE_DL_MAX") | $(get_emoji "$CHANGE_DL_AVG" "latency") $(format_change "$CHANGE_DL_AVG") | $(get_emoji "$CHANGE_DL_P90" "latency") $(format_change "$CHANGE_DL_P90") | $(get_emoji "$CHANGE_DL_P95" "latency") $(format_change "$CHANGE_DL_P95") | $(get_emoji "$CHANGE_DL_P99" "latency") $(format_change "$CHANGE_DL_P99") |
255+
| **Single Fetch** | $(get_emoji "$CHANGE_SF_MIN" "latency") $(format_change "$CHANGE_SF_MIN" "latency") | $(get_emoji "$CHANGE_SF_P50" "latency") $(format_change "$CHANGE_SF_P50" "latency") | $(get_emoji "$CHANGE_SF_MAX" "latency") $(format_change "$CHANGE_SF_MAX" "latency") | $(get_emoji "$CHANGE_SF_AVG" "latency") $(format_change "$CHANGE_SF_AVG" "latency") | $(get_emoji "$CHANGE_SF_P90" "latency") $(format_change "$CHANGE_SF_P90" "latency") | $(get_emoji "$CHANGE_SF_P95" "latency") $(format_change "$CHANGE_SF_P95" "latency") | $(get_emoji "$CHANGE_SF_P99" "latency") $(format_change "$CHANGE_SF_P99" "latency") |
256+
| **DataLoader** | $(get_emoji "$CHANGE_DL_MIN" "latency") $(format_change "$CHANGE_DL_MIN" "latency") | $(get_emoji "$CHANGE_DL_P50" "latency") $(format_change "$CHANGE_DL_P50" "latency") | $(get_emoji "$CHANGE_DL_MAX" "latency") $(format_change "$CHANGE_DL_MAX" "latency") | $(get_emoji "$CHANGE_DL_AVG" "latency") $(format_change "$CHANGE_DL_AVG" "latency") | $(get_emoji "$CHANGE_DL_P90" "latency") $(format_change "$CHANGE_DL_P90" "latency") | $(get_emoji "$CHANGE_DL_P95" "latency") $(format_change "$CHANGE_DL_P95" "latency") | $(get_emoji "$CHANGE_DL_P99" "latency") $(format_change "$CHANGE_DL_P99" "latency") |
243257
244258
### ⚡ Throughput
245259
246260
| Test | Metric | Current | Baseline | Change |
247261
|------|--------|---------|----------|--------|
248-
| **Single Fetch** | **Requests/sec** | ${CURRENT_SF_RPS} req/s | ${BASELINE_SF_RPS} req/s | $(get_emoji "$CHANGE_SF_RPS" "throughput") $(format_change "$CHANGE_SF_RPS") |
249-
| **DataLoader** | **Requests/sec** | ${CURRENT_DL_RPS} req/s | ${BASELINE_DL_RPS} req/s | $(get_emoji "$CHANGE_DL_RPS" "throughput") $(format_change "$CHANGE_DL_RPS") |
262+
| **Single Fetch** | **Requests/sec** | ${CURRENT_SF_RPS} req/s | ${BASELINE_SF_RPS} req/s | $(get_emoji "$CHANGE_SF_RPS" "throughput") $(format_change "$CHANGE_SF_RPS" "throughput") |
263+
| **DataLoader** | **Requests/sec** | ${CURRENT_DL_RPS} req/s | ${BASELINE_DL_RPS} req/s | $(get_emoji "$CHANGE_DL_RPS" "throughput") $(format_change "$CHANGE_DL_RPS" "throughput") |
250264
251265
### 🎯 Reliability
252266

‎src/HotChocolate/AspNetCore/benchmarks/k6/run-and-collect.sh‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -105,11 +105,14 @@ SINGLE_P95=$(extract_metric /tmp/single-fetch-summary.json "http_req_duration{ph
105105
SINGLE_P99=$(extract_metric /tmp/single-fetch-summary.json "http_req_duration{phase:measurement}" "p(99)")
106106
[ "$SINGLE_P99" == "0" ] && SINGLE_P99=$(extract_metric /tmp/single-fetch-summary.json "http_req_duration" "p(99)")
107107

108-
SINGLE_RPS=$(extract_metric /tmp/single-fetch-summary.json "http_reqs" "rate")
108+
SINGLE_RPS=$(extract_metric /tmp/single-fetch-summary.json "http_reqs{phase:measurement}" "rate")
109+
[ "$SINGLE_RPS" == "0" ] && SINGLE_RPS=$(extract_metric /tmp/single-fetch-summary.json "http_reqs" "rate")
110+
109111
SINGLE_ERROR_RATE=$(extract_metric /tmp/single-fetch-summary.json "http_req_failed{phase:measurement}" "rate")
110112
[ "$SINGLE_ERROR_RATE" == "0" ] && SINGLE_ERROR_RATE=$(extract_metric /tmp/single-fetch-summary.json "http_req_failed" "rate")
111113

112-
SINGLE_ITERATIONS=$(extract_metric /tmp/single-fetch-summary.json "iterations" "count")
114+
SINGLE_ITERATIONS=$(extract_metric /tmp/single-fetch-summary.json "iterations{phase:measurement}" "count")
115+
[ "$SINGLE_ITERATIONS" == "0" ] && SINGLE_ITERATIONS=$(extract_metric /tmp/single-fetch-summary.json "iterations" "count")
113116

114117
# Extract metrics from dataloader test
115118
DATALOADER_MIN=$(extract_metric /tmp/dataloader-summary.json "http_req_duration{phase:measurement}" "min")
@@ -133,11 +136,14 @@ DATALOADER_P95=$(extract_metric /tmp/dataloader-summary.json "http_req_duration{
133136
DATALOADER_P99=$(extract_metric /tmp/dataloader-summary.json "http_req_duration{phase:measurement}" "p(99)")
134137
[ "$DATALOADER_P99" == "0" ] && DATALOADER_P99=$(extract_metric /tmp/dataloader-summary.json "http_req_duration" "p(99)")
135138

136-
DATALOADER_RPS=$(extract_metric /tmp/dataloader-summary.json "http_reqs" "rate")
139+
DATALOADER_RPS=$(extract_metric /tmp/dataloader-summary.json "http_reqs{phase:measurement}" "rate")
140+
[ "$DATALOADER_RPS" == "0" ] && DATALOADER_RPS=$(extract_metric /tmp/dataloader-summary.json "http_reqs" "rate")
141+
137142
DATALOADER_ERROR_RATE=$(extract_metric /tmp/dataloader-summary.json "http_req_failed{phase:measurement}" "rate")
138143
[ "$DATALOADER_ERROR_RATE" == "0" ] && DATALOADER_ERROR_RATE=$(extract_metric /tmp/dataloader-summary.json "http_req_failed" "rate")
139144

140-
DATALOADER_ITERATIONS=$(extract_metric /tmp/dataloader-summary.json "iterations" "count")
145+
DATALOADER_ITERATIONS=$(extract_metric /tmp/dataloader-summary.json "iterations{phase:measurement}" "count")
146+
[ "$DATALOADER_ITERATIONS" == "0" ] && DATALOADER_ITERATIONS=$(extract_metric /tmp/dataloader-summary.json "iterations" "count")
141147

142148
# Create JSON output
143149
cat > "$OUTPUT_FILE" <<EOF

0 commit comments

Comments
 (0)