Skip to content

Commit 2f182b7

Browse files
michaelstaibmichaelstaib
authored
Updated Benchmarking Runner (#8859)
1 parent 8883074 commit 2f182b7

File tree

3 files changed

+172
-296
lines changed

3 files changed

+172
-296
lines changed

‎.github/workflows/benchmarks.yml‎

Lines changed: 151 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -1,115 +1,158 @@
1-
# This workflow uses actions that are not certified by GitHub.
2-
# They are provided by a third-party and are governed by
3-
# separate terms of service, privacy policy, and support
4-
# documentation.
5-
6-
# This workflow will build, test, sign and package a WPF or Windows Forms desktop application
7-
# built on .NET Core.
8-
# To learn how to migrate your existing application to .NET Core,
9-
# refer to https://docs.microsoft.com/en-us/dotnet/desktop-wpf/migration/convert-project-from-net-framework
10-
#
11-
# To configure this workflow:
12-
#
13-
# 1. Configure environment variables
14-
# GitHub sets default environment variables for every workflow run.
15-
# Replace the variables relative to your project in the "env" section below.
16-
#
17-
# 2. Signing
18-
# Generate a signing certificate in the Windows Application
19-
# Packaging Project or add an existing signing certificate to the project.
20-
# Next, use PowerShell to encode the .pfx file using Base64 encoding
21-
# by running the following Powershell script to generate the output string:
22-
#
23-
# $pfx_cert = Get-Content '.\SigningCertificate.pfx' -Encoding Byte
24-
# [System.Convert]::ToBase64String($pfx_cert) | Out-File 'SigningCertificate_Encoded.txt'
25-
#
26-
# Open the output file, SigningCertificate_Encoded.txt, and copy the
27-
# string inside. Then, add the string to the repo as a GitHub secret
28-
# and name it "Base64_Encoded_Pfx."
29-
# For more information on how to configure your signing certificate for
30-
# this workflow, refer to https://github.com/microsoft/github-actions-for-desktop-apps#signing
31-
#
32-
# Finally, add the signing certificate password to the repo as a secret and name it "Pfx_Key".
33-
# See "Build the Windows Application Packaging project" below to see how the secret is used.
34-
#
35-
# For more information on GitHub Actions, refer to https://github.com/features/actions
36-
# For a complete CI/CD sample to get started with GitHub Action workflows for Desktop Applications,
37-
# refer to https://github.com/microsoft/github-actions-for-desktop-apps
38-
39-
name: .NET Core Desktop
1+
name: Benchmarks
402

413
on:
42-
push:
43-
branches: [ "main" ]
444
pull_request:
45-
branches: [ "main" ]
5+
types: [opened, synchronize, reopened, ready_for_review]
6+
branches:
7+
- main
8+
- main-version-*
9+
paths-ignore:
10+
- "src/HotChocolate/AspNetCore/benchmarks/k6/performance-data.json"
4611

47-
jobs:
48-
49-
build:
50-
51-
strategy:
52-
matrix:
53-
configuration: [Debug, Release]
12+
concurrency:
13+
group: benchmarks-${{ github.event.pull_request.number }}
14+
cancel-in-progress: true
5415

55-
runs-on: windows-latest # For a list of available runner types, refer to
56-
# https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idruns-on
57-
58-
env:
59-
Solution_Name: your-solution-name # Replace with your solution name, i.e. MyWpfApp.sln.
60-
Test_Project_Path: your-test-project-path # Replace with the path to your test project, i.e. MyWpfApp.Tests\MyWpfApp.Tests.csproj.
61-
Wap_Project_Directory: your-wap-project-directory-name # Replace with the Wap project directory relative to the solution, i.e. MyWpfApp.Package.
62-
Wap_Project_Path: your-wap-project-path # Replace with the path to your Wap project, i.e. MyWpf.App.Package\MyWpfApp.Package.wapproj.
16+
jobs:
17+
hotchocolate-core:
18+
name: "HotChocolate Core Benchmarks"
19+
if: github.event.pull_request.draft == false
20+
runs-on: benchmarking
21+
permissions:
22+
contents: write
23+
pull-requests: write
6324

6425
steps:
65-
- name: Checkout
66-
uses: actions/checkout@v4
67-
with:
68-
fetch-depth: 0
69-
70-
# Install the .NET Core workload
71-
- name: Install .NET Core
72-
uses: actions/setup-dotnet@v4
73-
with:
74-
dotnet-version: 8.0.x
75-
76-
# Add MSBuild to the PATH: https://github.com/microsoft/setup-msbuild
77-
- name: Setup MSBuild.exe
78-
uses: microsoft/setup-msbuild@v2
79-
80-
# Execute all unit tests in the solution
81-
- name: Execute unit tests
82-
run: dotnet test
83-
84-
# Restore the application to populate the obj folder with RuntimeIdentifiers
85-
- name: Restore the application
86-
run: msbuild $env:Solution_Name /t:Restore /p:Configuration=$env:Configuration
87-
env:
88-
Configuration: ${{ matrix.configuration }}
89-
90-
# Decode the base 64 encoded pfx and save the Signing_Certificate
91-
- name: Decode the pfx
92-
run: |
93-
$pfx_cert_byte = [System.Convert]::FromBase64String("${{ secrets.Base64_Encoded_Pfx }}")
94-
$certificatePath = Join-Path -Path $env:Wap_Project_Directory -ChildPath GitHubActionsWorkflow.pfx
95-
[IO.File]::WriteAllBytes("$certificatePath", $pfx_cert_byte)
96-
97-
# Create the app package by building and packaging the Windows Application Packaging project
98-
- name: Create the app package
99-
run: msbuild $env:Wap_Project_Path /p:Configuration=$env:Configuration /p:UapAppxPackageBuildMode=$env:Appx_Package_Build_Mode /p:AppxBundle=$env:Appx_Bundle /p:PackageCertificateKeyFile=GitHubActionsWorkflow.pfx /p:PackageCertificatePassword=${{ secrets.Pfx_Key }}
100-
env:
101-
Appx_Bundle: Always
102-
Appx_Bundle_Platforms: x86|x64
103-
Appx_Package_Build_Mode: StoreUpload
104-
Configuration: ${{ matrix.configuration }}
105-
106-
# Remove the pfx
107-
- name: Remove the pfx
108-
run: Remove-Item -path $env:Wap_Project_Directory\GitHubActionsWorkflow.pfx
109-
110-
# Upload the MSIX package: https://github.com/marketplace/actions/upload-a-build-artifact
111-
- name: Upload build artifacts
112-
uses: actions/upload-artifact@v4
113-
with:
114-
name: MSIX Package
115-
path: ${{ env.Wap_Project_Directory }}\AppPackages
26+
- name: Checkout PR code
27+
uses: actions/checkout@v4
28+
with:
29+
fetch-depth: 0
30+
show-progress: false
31+
32+
- name: Restore dependencies
33+
run: dotnet restore src/HotChocolate/AspNetCore/benchmarks/k6/eShop.slnx
34+
35+
- name: Start AppHost and wait for readiness
36+
working-directory: src/HotChocolate/AspNetCore/benchmarks/k6/Catalog.AppHost
37+
run: |
38+
echo "Starting AppHost..."
39+
dotnet run > /tmp/apphost.log 2>&1 &
40+
APPHOST_PID=$!
41+
echo "APPHOST_PID=$APPHOST_PID" >> $GITHUB_ENV
42+
43+
echo "Waiting for server to be ready..."
44+
for i in {1..30}; do
45+
if curl -s -o /dev/null -w "%{http_code}" http://localhost:5224/graphql -X POST \
46+
-H "Content-Type: application/json" \
47+
-d '{"query": "{ __typename }"}' | grep -q "200"; then
48+
echo "Server is ready!"
49+
break
50+
fi
51+
echo "Waiting... ($i/30)"
52+
sleep 2
53+
done
54+
55+
- name: Run performance tests and collect data
56+
working-directory: src/HotChocolate/AspNetCore/benchmarks/k6
57+
run: |
58+
chmod +x run-and-collect.sh
59+
./run-and-collect.sh performance-data-current.json
60+
61+
- name: Stop AppHost
62+
if: always()
63+
run: |
64+
if [ -n "$APPHOST_PID" ]; then
65+
kill $APPHOST_PID 2>/dev/null || true
66+
wait $APPHOST_PID 2>/dev/null || true
67+
fi
68+
69+
- name: Commit and push performance data to current branch
70+
working-directory: src/HotChocolate/AspNetCore/benchmarks/k6
71+
run: |
72+
# Copy the performance data to the tracked filename
73+
cp performance-data-current.json performance-data.json
74+
75+
# Configure git
76+
git config user.name "github-actions[bot]"
77+
git config user.email "github-actions[bot]@users.noreply.github.com"
78+
79+
# Add and commit the performance data
80+
git add performance-data.json
81+
82+
# Only commit if there are changes
83+
if ! git diff --staged --quiet; then
84+
git commit -m "Update performance data [skip ci]"
85+
git push origin HEAD:${{ github.head_ref }}
86+
else
87+
echo "No changes to performance data"
88+
fi
89+
90+
- name: Fetch baseline performance data from main
91+
run: |
92+
git fetch origin main:main
93+
if git show main:src/HotChocolate/AspNetCore/benchmarks/k6/performance-data.json > baseline-performance.json 2>/dev/null; then
94+
echo "Baseline data fetched successfully"
95+
cat baseline-performance.json
96+
else
97+
echo "No baseline data found on main branch"
98+
# Don't create the file - let the comparison script handle missing baseline
99+
rm -f baseline-performance.json
100+
fi
101+
102+
- name: Compare performance and generate report
103+
working-directory: src/HotChocolate/AspNetCore/benchmarks/k6
104+
run: |
105+
chmod +x compare-performance.sh
106+
./compare-performance.sh performance-data-current.json ../../../../../baseline-performance.json performance-report.md
107+
108+
- name: Comment PR with performance report
109+
uses: actions/github-script@v7
110+
with:
111+
github-token: ${{ secrets.GITHUB_TOKEN }}
112+
script: |
113+
const fs = require('fs');
114+
const reportPath = 'src/HotChocolate/AspNetCore/benchmarks/k6/performance-report.md';
115+
116+
let report;
117+
try {
118+
report = fs.readFileSync(reportPath, 'utf8');
119+
} catch (error) {
120+
console.error('Failed to read performance report:', error);
121+
return;
122+
}
123+
124+
// Add timestamp and commit info to the report
125+
const timestamp = new Date().toUTCString();
126+
const commitSha = context.sha.substring(0, 7);
127+
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
128+
129+
const commentBody = `${report}\n\n---\n*Run [${context.runId}](${runUrl}) • Commit ${commitSha} • ${timestamp}*`;
130+
131+
// Always create a new comment
132+
await github.rest.issues.createComment({
133+
owner: context.repo.owner,
134+
repo: context.repo.repo,
135+
issue_number: context.issue.number,
136+
body: commentBody,
137+
});
138+
139+
- name: Upload performance data as artifact
140+
uses: actions/upload-artifact@v4
141+
if: always()
142+
with:
143+
name: performance-data
144+
path: |
145+
src/HotChocolate/AspNetCore/benchmarks/k6/performance-data-current.json
146+
src/HotChocolate/AspNetCore/benchmarks/k6/performance-report.md
147+
/tmp/apphost.log
148+
retention-days: 30
149+
150+
- name: Check for performance regression
151+
working-directory: src/HotChocolate/AspNetCore/benchmarks/k6
152+
run: |
153+
# Fail the build if there's a significant performance regression
154+
if grep -q "⚠️ \*\*Performance regression detected" performance-report.md; then
155+
echo "::warning::Performance regression detected! Please review the performance report."
156+
# Uncomment the next line to fail the build on regression
157+
# exit 1
158+
fi

0 commit comments

Comments
 (0)