Skip to content

Commit b2cde32

Browse files
zhangxinyuehfadzhangxinyuehfad
committed
[Test] Add accuracy test report
Signed-off-by: hfadzxy <starmoon_zhang@163.com>
1 parent fa4a5d9 commit b2cde32

File tree

6 files changed

+548
-2
lines changed

6 files changed

+548
-2
lines changed

‎.github/actionlint.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@ self-hosted-runner:
22
# Labels of self-hosted runner in array of strings.
33
labels:
44
- linux-arm64-npu-1
5+
- linux-arm64-npu-2
56
- linux-arm64-npu-4

‎.github/workflows/accuracy_report.yaml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
#
2+
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# This file is a part of the vllm-ascend project.
16+
#
17+
18+
name: Accuracy Report
19+
on:
20+
workflow_run:
21+
workflows: ["accuarcy test"]
22+
types: [completed]
23+
workflow_dispatch:
24+
inputs:
25+
branch:
26+
description: 'choose a dev branch to pr'
27+
required: true
28+
vllm-ascend-version:
29+
description: 'what vllm-ascend version to accuracy test?'
30+
required: true
31+
type: string
32+
jobs:
33+
download:
34+
runs-on: ubuntu-latest
35+
steps:
36+
- name: Checkout repository
37+
uses: actions/checkout@v4
38+
with:
39+
ref: ${{ github.event.inputs.branch }}
40+
41+
- name: Debug List Artifacts
42+
run: gh api /repos/${{ github.repository }}/actions/artifacts
43+
env:
44+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
45+
46+
- name: Query artifact run id for Qwen2.5-VL-7B-Instruct V0 latest artifact
47+
id: get_Qwen2_5_VL_7B_Instruct_latest_run_id_V0
48+
run: |
49+
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
50+
RUN_ID=$(echo "$ARTIFACT_JSON" | \
51+
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen2.5-VL-7B-Instruct-V0-report")] | sort_by(.created_at) | last | .workflow_run.id')
52+
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
53+
env:
54+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
55+
56+
- name: Query artifact run id for Qwen2.5-7B-Instruct V0 latest artifact
57+
id: get_Qwen2_5_7B_Instruct_latest_run_id_V0
58+
run: |
59+
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
60+
RUN_ID=$(echo "$ARTIFACT_JSON" | \
61+
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Qwen2.5-7B-Instruct-V0-report")] | sort_by(.created_at) | last | .workflow_run.id')
62+
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
63+
env:
64+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
65+
66+
- name: Query artifact run id for Llama-3.1-8B-Instruct V0 latest artifact
67+
id: get_Llama_3_1_8B_Instruct_latest_run_id_V0
68+
run: |
69+
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
70+
RUN_ID=$(echo "$ARTIFACT_JSON" | \
71+
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.vllm-ascend-version }}-Llama-3.1-8B-Instruct-V0-report")] | sort_by(.created_at) | last | .workflow_run.id')
72+
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
73+
env:
74+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
75+
76+
- name: Download Qwen/Qwen2.5-VL-7B-Instruct V0 Artifact
77+
uses: actions/download-artifact@v4
78+
with:
79+
name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen2.5-VL-7B-Instruct-V0-report
80+
path: ./docs/source/developer_guide/evaluation/accuracy_report
81+
github-token: ${{ secrets.GITHUB_TOKEN }}
82+
repository: vllm-project/vllm-ascend
83+
run-id: ${{ steps.get_Qwen2_5_VL_7B_Instruct_latest_run_id_V0.outputs.runid }}
84+
85+
- name: Download Qwen/Qwen2.5-7B-Instruct Artifact
86+
uses: actions/download-artifact@v4
87+
with:
88+
name: ${{ github.event.inputs.vllm-ascend-version }}-Qwen2.5-7B-Instruct-V0-report
89+
path: ./docs/source/developer_guide/evaluation/accuracy_report
90+
github-token: ${{ secrets.GITHUB_TOKEN }}
91+
repository: vllm-project/vllm-ascend
92+
run-id: ${{ steps.get_Qwen2_5_7B_Instruct_latest_run_id_V0.outputs.runid }}
93+
94+
- name: Download meta-llama/Llama-3.1-8B-Instruct Artifact
95+
uses: actions/download-artifact@v4
96+
with:
97+
name: ${{ github.event.inputs.vllm-ascend-version }}-Llama-3.1-8B-Instruct-V0-report
98+
path: ./docs/source/developer_guide/evaluation/accuracy_report
99+
github-token: ${{ secrets.GITHUB_TOKEN }}
100+
repository: vllm-project/vllm-ascend
101+
run-id: ${{ steps.get_Llama_3_1_8B_Instruct_latest_run_id_V0.outputs.runid }}
102+
103+
- name: Display Files
104+
working-directory: ./docs/source/developer_guide/evaluation/accuracy_report
105+
run: |
106+
cat ./Qwen2.5-VL-7B-Instruct.md
107+
cat ./Llama-3.1-8B-Instruct.md
108+
cat ./Qwen2.5-7B-Instruct.md
109+
110+
- name: Create Pull Request for markdown update
111+
uses: peter-evans/create-pull-request@v7
112+
with:
113+
token: ${{ secrets.PR_TOKEN }}
114+
base: ${{ github.ref_name }}
115+
branch: auto-pr/accuracy-test
116+
commit-message: "Update accuracy report for ${{ github.event.inputs.branch }}"
117+
add-paths: ./docs/source/developer_guide/evaluation/accuracy_report/*.md
118+
title: "[Doc]Update accuracy report for ${{ github.event.inputs.branch }}"
119+
body: |
120+
The accuracy results running on Ascend NPU have changed, I'm updating the report.
121+
Please review the changes.
122+
123+
- [Workflow run][1]
124+
- [Qwen2.5-7B-Instruct accuracy report][2]
125+
- [Llama-3.1-8B-Instruct accuracy report][3]
126+
- [Qwen2.5-VL-7B-Instruct accuracy report][4]
127+
128+
[1]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
129+
[2]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_7B_Instruct_latest_run_id_V0.outputs.runid }}
130+
[3]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Llama_3_1_8B_Instruct_latest_run_id_V0.outputs.runid }}
131+
[4]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_VL_7B_Instruct_latest_run_id_V0.outputs.runid }}

‎.github/workflows/accuracy_test.yaml‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
#
2+
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# This file is a part of the vllm-ascend project.
16+
#
17+
18+
name: Accuracy Tests
19+
20+
21+
on:
22+
pull_request:
23+
types: [opened, synchronize, reopened]
24+
25+
defaults:
26+
run:
27+
shell: bash -el {0}
28+
29+
jobs:
30+
model_tests:
31+
name: Model Test - ${{ matrix.output_file }}
32+
runs-on: 'linux-arm64-npu-2'
33+
34+
strategy:
35+
fail-fast: false
36+
matrix:
37+
include:
38+
- model_name: "Qwen/Qwen2.5-VL-7B-Instruct"
39+
output_file: "Qwen2.5-VL-7B-Instruct"
40+
41+
container:
42+
image: quay.io/ascend/cann:8.0.0-910b-ubuntu22.04-py3.10
43+
env:
44+
HF_ENDPOINT: https://hf-mirror.com
45+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
46+
DATASET_SOURCE: ModelScope
47+
48+
49+
steps:
50+
- name: Checkout repository
51+
uses: actions/checkout@v4
52+
53+
- name: Check npu and CANN info
54+
run: |
55+
npu-smi info
56+
cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
57+
58+
- name: Config mirrors
59+
run: |
60+
sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
61+
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
62+
apt-get update -y
63+
apt install git -y
64+
git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf https://github.com/
65+
66+
- name: Install system dependencies
67+
run: |
68+
apt-get -y install `cat packages.txt`
69+
apt-get -y install gcc g++ cmake libnuma-dev
70+
71+
72+
- name: Install system dependencies
73+
run: |
74+
apt-get -y install `cat packages.txt`
75+
apt-get -y install gcc g++ cmake libnuma-dev
76+
77+
- name: Checkout vllm-project/vllm repo
78+
uses: actions/checkout@v4
79+
with:
80+
repository: vllm-project/vllm
81+
path: ./vllm-empty
82+
83+
- name: Install vllm-project/vllm from source
84+
working-directory: ./vllm-empty
85+
run: VLLM_TARGET_DEVICE=empty pip install -e .
86+
87+
- name: Install pta
88+
run: |
89+
if [ ! -d /root/.cache/pta ]; then
90+
mkdir -p /root/.cache/pta
91+
fi
92+
if [ ! -f /root/.cache/pta/torch_npu-2.5.1.dev20250320-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl ]; then
93+
cd /root/.cache/pta
94+
rm -rf pytorch_v2.5.1_py310*
95+
wget https://pytorch-package.obs.cn-north-4.myhuaweicloud.com/pta/Daily/v2.5.1/20250320.3/pytorch_v2.5.1_py310.tar.gz
96+
tar -zxvf pytorch_v2.5.1_py310.tar.gz
97+
fi
98+
pip install /root/.cache/pta/torch_npu-2.5.1.dev20250320-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
99+
100+
- name: Checkout vllm-project/vllm-ascend repo
101+
uses: actions/checkout@v4
102+
with:
103+
repository: vllm-project/vllm-ascend
104+
path: ./vllm-ascend
105+
fetch-depth: 0
106+
107+
- name: Install vllm-project/vllm-ascend
108+
working-directory: ./vllm-ascend
109+
run: |
110+
pip install -r requirements-dev.txt
111+
pip install -e .
112+
113+
- name: Checkout EleutherAI/lm-evaluation-harness repo
114+
uses: actions/checkout@v4
115+
with:
116+
repository: EleutherAI/lm-evaluation-harness
117+
path: ./lm-eval
118+
fetch-depth: 0
119+
120+
- name: Install EleutherAI/lm-evaluation-harness
121+
working-directory: ./lm-eval
122+
run: |
123+
pip install -e .
124+
pip install ray datasets==2.16.0 transformers==4.50.3 huggingface-hub==0.29.3
125+
126+
- name: Collect version info
127+
run: |
128+
for dir in /usr/local/Ascend/ascend-toolkit/*; do
129+
dname=$(basename "$dir")
130+
if [ "$dname" != "latest" ]; then
131+
TOOLKIT_DIR="$dname"
132+
break
133+
fi
134+
done
135+
INFO_FILE="/usr/local/Ascend/ascend-toolkit/${TOOLKIT_DIR}/$(uname -i)-linux/ascend_toolkit_install.info"
136+
CANN_VERSION=$(grep "version=" "$INFO_FILE" \
137+
| head -n1 \
138+
| cut -d'=' -f2 \
139+
| tr -d '"')
140+
{
141+
echo "CANN_VERSION=$CANN_VERSION"
142+
pip show torch | grep "Version:" | awk '{print "TORCH_VERSION="$2}'
143+
pip show torch_npu | grep "Version:" | awk '{print "TORCH_NPU_VERSION="$2}'
144+
pip show vllm | grep "Version:" | awk '{print "VLLM_VERSION="$2}' | sed 's/+.*//'
145+
} >> "$GITHUB_ENV"
146+
147+
- name: Print versions
148+
run: |
149+
echo "CANN: ${{ env.CANN_VERSION }}"
150+
echo "Torch NPU: ${{ env.TORCH_NPU_VERSION }}"
151+
echo "Torch: ${{ env.TORCH_VERSION }}"
152+
echo "vLLM: ${{ env.VLLM_VERSION }}"
153+
154+
- name: Run Accuracy Test for V0
155+
working-directory: ./benchmarks
156+
env:
157+
VLLM_USE_V1: 0
158+
PYTORCH_NPU_ALLOC_CONF: max_split_size_mb:256
159+
run: |
160+
mkdir -p ./accuracy/V0
161+
python ./scripts/run_accuracy.py \
162+
--model "${{ matrix.model_name }}" \
163+
--output "./accuracy/V0/${{ matrix.output_file }}.md" \
164+
--vllm_ascend_version "test1" \
165+
--cann_version "${{ env.CANN_VERSION }}" \
166+
--torch_npu_version "${{ env.TORCH_NPU_VERSION }}" \
167+
--torch_version "${{ env.TORCH_VERSION }}" \
168+
--vllm_version "${{ env.VLLM_VERSION }}"
169+
170+
- name: Upload Report for V0
171+
uses: actions/upload-artifact@v4
172+
with:
173+
name: "${{ github.event.inputs.vllm-ascend-version }}-${{ matrix.output_file }}-V0-report"
174+
path: ./benchmarks/accuracy/V0/${{ matrix.output_file }}.md
175+
if-no-files-found: warn
176+
retention-days: 90
177+
overwrite: true

0 commit comments

Comments
 (0)