Skip to content

Commit bdc6370

Browse files
zhangxinyuehfadzhangxinyuehfad
committed
[CI] Update accuracy CI
Signed-off-by: hfadzxy <starmoon_zhang@163.com>
1 parent f39bd30 commit bdc6370

File tree

2 files changed

+280
-4
lines changed

2 files changed

+280
-4
lines changed

‎.github/workflows/accuracy_ci.yaml‎

Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
#
2+
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# This file is a part of the vllm-ascend project.
16+
#
17+
name: Benchmarks / accuracy-ci
18+
19+
on:
20+
pull_request:
21+
branches:
22+
- 'main'
23+
- '*-dev'
24+
paths:
25+
- '.github/workflows/accuracy_ci.yaml'
26+
- 'tests/e2e/models/**'
27+
schedule:
28+
- cron: '0 */6 * * *'
29+
30+
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
31+
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
32+
# It's used to activate ascend-toolkit environment variables.
33+
defaults:
34+
run:
35+
shell: bash -el {0}
36+
37+
# only cancel in-progress runs of the same workflow
38+
concurrency:
39+
group: ${{ github.workflow }}-${{ github.ref }}
40+
cancel-in-progress: true
41+
42+
jobs:
43+
prepare_matrix:
44+
runs-on: ubuntu-latest
45+
outputs:
46+
matrix: ${{ steps.set_matrix.outputs.matrix }}
47+
steps:
48+
- name: Checkout repository
49+
uses: actions/checkout@v4
50+
with:
51+
fetch-depth: 0
52+
53+
- name: Install dependencies
54+
run: |
55+
sudo apt-get update
56+
sudo apt-get install -y jq
57+
pip install yq
58+
59+
- name: Set mappings
60+
run: |
61+
mappings_json='{}'
62+
63+
for config_file in tests/e2e/models/configs/*.yaml; do
64+
echo "Processing config file: $config_file"
65+
full_model_name=$(yq -r '.model_name' "$config_file" | tr -d '"')
66+
model_name=$(echo "$full_model_name" | awk -F'/' '{print $NF}')
67+
runner=$(yq -r '.runner' "$config_file" | tr -d '"')
68+
69+
echo "Raw model_name: $full_model_name"
70+
echo "Extracted model_name: $model_name"
71+
echo "Runner: $runner"
72+
73+
mappings_json=$(echo "$mappings_json" | jq --arg key "$model_name" --arg value "$runner" '. + {($key): $value}')
74+
done
75+
76+
compact_json=$(echo "$mappings_json" | jq -c .)
77+
78+
echo "Generated mappings:"
79+
echo "$mappings_json"
80+
echo "MAPPINGS_JSON=$compact_json" >> $GITHUB_ENV
81+
82+
- name: Prepare matrix
83+
id: set_matrix
84+
shell: bash
85+
env:
86+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
87+
run: |
88+
set -e
89+
event="${GITHUB_EVENT_NAME}"
90+
matrix="[]"
91+
92+
if [[ "$event" == "schedule" ]]; then
93+
matrix=$(echo "$MAPPINGS_JSON" | jq '[to_entries[] | {model_name: .key, runner: .value}]')
94+
else
95+
pr_number=$(jq -r '.pull_request.number // empty' "$GITHUB_EVENT_PATH" || true)
96+
97+
changed_files=""
98+
echo "PR detected: #$pr_number — fetching changed files via API"
99+
changed_files=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" \
100+
"$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/pulls/$pr_number/files?per_page=100" \
101+
| jq -r '.[].filename' || true)
102+
103+
echo "Changed files:"
104+
echo "$changed_files"
105+
106+
run_all=false
107+
108+
for f in $changed_files; do
109+
if [[ "$f" != tests/e2e/models/configs/* ]]; then
110+
echo "Non-config file changed: $f, will run all models"
111+
run_all=true
112+
break
113+
fi
114+
done
115+
116+
if [[ "$run_all" == "true" ]]; then
117+
matrix=$(echo "$MAPPINGS_JSON" | jq '[to_entries[] | {model_name: .key, runner: .value}]')
118+
else
119+
for f in $changed_files; do
120+
if [[ "$f" == tests/e2e/models/configs/*.yaml ]]; then
121+
name=$(basename "$f" .yaml)
122+
runner=$(echo "$MAPPINGS_JSON" | jq -r --arg key "$name" '.[$key] // empty')
123+
if [[ -n "$runner" ]]; then
124+
matrix=$(echo "$matrix" | jq --arg model "$name" --arg runner "$runner" '. += [{"model_name":$model, "runner":$runner}]')
125+
else
126+
echo "Config $name not found in mappings; skipping."
127+
fi
128+
fi
129+
done
130+
fi
131+
fi
132+
133+
echo "Generated matrix (raw): $matrix"
134+
compact_matrix=$(echo "$matrix" | jq -c 'if type=="array" then . else [] end')
135+
echo "matrix=$compact_matrix" >> $GITHUB_OUTPUT
136+
echo "Final matrix output: $compact_matrix"
137+
138+
accuracy_tests:
139+
needs: prepare_matrix
140+
runs-on: ${{ matrix.runner }}
141+
strategy:
142+
matrix:
143+
include: ${{ fromJson(needs.prepare_matrix.outputs.matrix) }}
144+
fail-fast: false
145+
name: ${{ matrix.model_name }} accuracy
146+
container:
147+
image: >-
148+
${{
149+
contains(matrix.runner, '310p')
150+
&& 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-310p-ubuntu22.04-py3.11'
151+
|| 'swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-910b-ubuntu22.04-py3.11'
152+
}}
153+
env:
154+
VLLM_USE_MODELSCOPE: True
155+
156+
steps:
157+
- name: Checkout repository
158+
uses: actions/checkout@v4
159+
160+
- name: Set model name as output
161+
id: set_output
162+
run: |
163+
echo "model_name=${{ matrix.model_name }}" >> $GITHUB_OUTPUT
164+
165+
- name: Config mirrors
166+
run: |
167+
sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list
168+
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
169+
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
170+
apt-get update -y
171+
apt install git -y
172+
173+
- name: Install system dependencies
174+
run: |
175+
apt-get -y install `cat packages.txt`
176+
apt-get -y install gcc g++ cmake libnuma-dev
177+
178+
- name: Checkout vllm-project/vllm repo
179+
uses: actions/checkout@v4
180+
with:
181+
repository: vllm-project/vllm
182+
ref: v0.10.2
183+
path: ./vllm-empty
184+
185+
- name: Install vllm-project/vllm from source
186+
working-directory: ./vllm-empty
187+
run: |
188+
VLLM_TARGET_DEVICE=empty pip install -e .
189+
190+
- name: Checkout vllm-project/vllm-ascend repo
191+
uses: actions/checkout@v4
192+
with:
193+
repository: vllm-project/vllm-ascend
194+
path: ./vllm-ascend
195+
196+
- name: Install vllm-project/vllm-ascend
197+
working-directory: ./vllm-ascend
198+
env:
199+
PIP_EXTRA_INDEX_URL: https://mirrors.huaweicloud.com/ascend/repos/pypi
200+
run: |
201+
runner="${{ matrix.runner }}"
202+
# If runner indicates 310p, set SOC_VERSION and LD_LIBRARY_PATH before install
203+
if [[ "$runner" == *310p* ]]; then
204+
export SOC_VERSION=ASCEND310P3
205+
fi
206+
pip install -r requirements-dev.txt
207+
pip install -v -e .
208+
209+
- name: Get vLLM commit hash and URL
210+
working-directory: ./vllm-empty
211+
run: |
212+
VLLM_COMMIT=$(git rev-parse --short=7 HEAD)
213+
echo "VLLM_COMMIT=$VLLM_COMMIT" >> $GITHUB_ENV
214+
215+
- name: Get vLLM-Ascend commit hash and URL
216+
working-directory: ./vllm-ascend
217+
run: |
218+
VLLM_ASCEND_COMMIT=$(git rev-parse --short=7 HEAD)
219+
echo "VLLM_ASCEND_COMMIT=$VLLM_ASCEND_COMMIT" >> $GITHUB_ENV
220+
221+
- name: Collect version info
222+
run: |
223+
for dir in /usr/local/Ascend/ascend-toolkit/*; do
224+
dname=$(basename "$dir")
225+
if [ "$dname" != "latest" ]; then
226+
TOOLKIT_DIR="$dname"
227+
break
228+
fi
229+
done
230+
INFO_FILE="/usr/local/Ascend/ascend-toolkit/${TOOLKIT_DIR}/$(uname -i)-linux/ascend_toolkit_install.info"
231+
GHA_CANN_VERSION=$(grep "version=" "$INFO_FILE" \
232+
| head -n1 \
233+
| cut -d'=' -f2 \
234+
| tr -d '"')
235+
{
236+
echo "GHA_CANN_VERSION=$GHA_CANN_VERSION"
237+
pip show torch | grep "Version:" | awk '{print "GHA_TORCH_VERSION="$2}'
238+
pip show torch_npu | grep "Version:" | awk '{print "GHA_TORCH_NPU_VERSION="$2}'
239+
pip show vllm | grep "Version:" | awk '{print "GHA_VLLM_VERSION="$2}' | sed 's/+.*//'
240+
} >> "$GITHUB_ENV"
241+
242+
- name: Run accuracy test
243+
id: report
244+
env:
245+
VLLM_WORKER_MULTIPROC_METHOD: spawn
246+
VLLM_USE_MODELSCOPE: True
247+
VLLM_VERSION: ${{ env.GHA_VLLM_VERSION }}
248+
VLLM_COMMIT: ${{ env.VLLM_COMMIT }}
249+
VLLM_ASCEND_VERSION: ${{ env.GHA_VLLM_ASCEND_VERSION || github.ref }}
250+
VLLM_ASCEND_COMMIT: ${{ env.VLLM_ASCEND_COMMIT }}
251+
CANN_VERSION: ${{ env.GHA_CANN_VERSION }}
252+
TORCH_VERSION: ${{ env.GHA_TORCH_VERSION }}
253+
TORCH_NPU_VERSION: ${{ env.GHA_TORCH_NPU_VERSION }}
254+
run: |
255+
model_base_name=$(basename ${{ matrix.model_name }})
256+
markdown_name="${model_base_name}"
257+
echo "markdown_name=$markdown_name" >> $GITHUB_OUTPUT
258+
mkdir -p ./benchmarks/accuracy
259+
pytest -sv ./tests/e2e/models/test_lm_eval_correctness.py \
260+
--config ./tests/e2e/models/configs/${{ matrix.model_name }}.yaml
261+
262+
- name: Generate step summary
263+
if: ${{ always() }}
264+
run: |
265+
cat ./benchmarks/accuracy/${{ steps.report.outputs.markdown_name }}.md >> $GITHUB_STEP_SUMMARY
266+
267+
- name: Sanitize version string for artifact naming
268+
run: |
269+
SAFE_VLLM_ASCEND_VERSION="${GHA_VLLM_ASCEND_VERSION//\//-}"
270+
echo "SAFE_VLLM_ASCEND_VERSION=$SAFE_VLLM_ASCEND_VERSION" >> "$GITHUB_ENV"
271+
272+
- name: Upload Report
273+
uses: actions/upload-artifact@v4
274+
with:
275+
name: "report-${{ env.SAFE_VLLM_ASCEND_VERSION }}-${{ steps.report.outputs.markdown_name }}"
276+
path: ./benchmarks/accuracy/${{ steps.report.outputs.markdown_name }}.md
277+
if-no-files-found: warn
278+
retention-days: 90
279+
overwrite: true

‎.github/workflows/accuracy_test.yaml‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,6 @@
2222
name: ascend test / accuracy
2323

2424
on:
25-
schedule:
26-
# Runs every 6 hours
27-
- cron: '0 */6 * * *'
2825
pull_request:
2926
types: [ labeled ]
3027
workflow_dispatch:
@@ -58,7 +55,7 @@ jobs:
5855
${{
5956
contains(github.event.pull_request.labels.*.name, 'accuracy-test') &&
6057
contains(github.event.pull_request.labels.*.name, 'ready-for-test') ||
61-
github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
58+
github.event_name == 'workflow_dispatch'
6259
}}
6360
runs-on: ${{ matrix.runner }}
6461
strategy:

0 commit comments

Comments
 (0)