Skip to content

Commit 9eed06a

Browse files
HuanyuZhangfacebook-github-bot
HuanyuZhang
authored and
facebook-github-bot
committed
Fix github action CI for GPU (#625)
Summary: Pull Request resolved: #625 Reviewed By: EnayatUllah Differential Revision: D53292685 fbshipit-source-id: 99c9c8e27dc946765ac86e750b8109ceb2423740
1 parent 36f7a34 commit 9eed06a

File tree

2 files changed

+119
-114
lines changed

2 files changed

+119
-114
lines changed

‎.github/workflows/ci_gpu.yml

Lines changed: 118 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -8,63 +8,66 @@ on:
88
branches:
99
- main
1010

11-
unittest_multi_gpu:
12-
runs-on: linux.4xlarge.nvidia.gpu
13-
steps:
14-
- name: Checkout
15-
uses: actions/checkout@v2
11+
jobs:
12+
# Temporarily disable this test since there is no server including multiple GPUs.
13+
# unittest_multi_gpu:
14+
# runs-on: 4-core-ubuntu-gpu-t4
15+
# steps:
16+
# - name: Checkout
17+
# uses: actions/checkout@v2
1618

17-
- name: Set up Python
18-
uses: actions/setup-python@v2
19-
with:
20-
python-version: 3.9
19+
# - name: Display Python version
20+
# run: python3 -c "import sys; print(sys.version)"
2121

22-
- name: Install dependencies
23-
run: |
24-
./scripts/install_via_pip.sh -c
22+
# - name: Set up Python
23+
# uses: actions/setup-python@v2
24+
# with:
25+
# python-version: '3.x'
2526

26-
- name: Run multi-GPU unit tests
27-
run: |
28-
nvidia-smi
29-
nvcc --version
30-
python -m unittest opacus.tests.multigpu_gradcheck.GradientComputationTest.test_gradient_correct
27+
# - name: Install dependencies
28+
# run: |
29+
# python -m pip install --upgrade pip
30+
# ./scripts/install_via_pip.sh -c
31+
32+
# - name: Run multi-GPU unit tests
33+
# run: |
34+
# python3 -m unittest opacus.tests.multigpu_gradcheck.GradientComputationTest.test_gradient_correct
3135

3236

3337
integrationtest_py39_torch_release_cuda:
34-
runs-on: ubuntu-latest
35-
container:
36-
# https://hub.docker.com/r/nvidia/cuda
37-
image: nvidia/cuda:12.3.1-base-ubuntu22.04
38-
options: --gpus all
39-
env:
40-
TZ: 'UTC'
38+
runs-on: 4-core-ubuntu-gpu-t4
4139
steps:
4240
- name: Checkout
4341
uses: actions/checkout@v2
4442

4543
- name: Set up Python
4644
uses: actions/setup-python@v2
4745
with:
48-
python-version: 3.9
46+
python-version: '3.9'
4947

5048
- name: Install dependencies
5149
run: |
52-
python -m pip install --upgrade pip
50+
python3 -m pip install --upgrade pip
5351
pip install pytest coverage coveralls
5452
./scripts/install_via_pip.sh -c
5553
56-
- name: Install CUDA toolkit and cuDNN
57-
run: |
58-
apt-get update
59-
apt-get install -y --no-install-recommends \
60-
cuda-toolkit-11-1 \
61-
libcudnn8=8.1.1.33-1+cuda11.1 \
62-
libcudnn8-dev=8.1.1.33-1+cuda11.1
54+
# Cuda dependency has already been installed when installing PyTorch, so no need to re-install it.
55+
# https://discuss.pytorch.org/t/should-i-install-the-extra-cudatoolkit-and-cudnn/194528
56+
# Cuda installation guide: https://medium.com/@milistu/how-to-install-cuda-cudnn-7e4a00ae4f44
57+
# - name: Install CUDA toolkit and cuDNN
58+
# run: |
59+
# sudo apt-get update
60+
# sudo apt-get install -y --no-install-recommends \
61+
# cuda-toolkit-11-1 \
62+
# libcudnn8=8.1.1.33-1+cuda11.1 \
63+
# libcudnn8-dev=8.1.1.33-1+cuda11.1
6364

6465
- name: Run MNIST integration test (CUDA)
6566
run: |
67+
nvidia-smi
6668
mkdir -p runs/mnist/data
6769
mkdir -p runs/mnist/test-reports
70+
python -c "import torch; exit(0) if torch.cuda.is_available() else exit(1)"
6871
python examples/mnist.py --lr 0.25 --sigma 0.7 -c 1.5 --batch-size 64 --epochs 1 --data-root runs/mnist/data --n-runs 1 --device cuda
6972
python -c "import torch; accuracy = torch.load('run_results_mnist_0.25_0.7_1.5_64_1.pt'); exit(0) if (accuracy[0]>0.78 and accuracy[0]<0.95) else exit(1)"
7073
@@ -91,83 +94,85 @@ on:
9194
name: cifar10-gpu-reports
9295
path: runs/cifar10/test-reports
9396

94-
- name: Run IMDb integration test (CUDA)
95-
run: |
96-
mkdir -p runs/imdb/data
97-
mkdir -p runs/imdb/test-reports
98-
pip install --user datasets transformers
99-
python examples/imdb.py --lr 0.02 --sigma 1.0 -c 1.0 --batch-size 64 --max-sequence-length 256 --epochs 2 --data-root runs/imdb/data --device cuda
100-
python -c "import torch; accuracy = torch.load('run_results_imdb_classification.pt'); exit(0) if (accuracy>0.54 and accuracy<0.66) else exit(1)"
101-
102-
- name: Store IMDb test results
103-
uses: actions/upload-artifact@v2
104-
with:
105-
name: imdb-gpu-reports
106-
path: runs/imdb/test-reports
107-
108-
- name: Run charlstm integration test (CUDA)
109-
run: |
110-
mkdir -p runs/charlstm/data
111-
wget https://download.pytorch.org/tutorial/data.zip -O runs/charlstm/data/data.zip
112-
unzip runs/charlstm/data/data.zip -d runs/charlstm/data
113-
rm runs/charlstm/data/data.zip
114-
mkdir -p runs/charlstm/test-reports
115-
pip install scikit-learn
116-
python examples/char-lstm-classification.py --epochs=20 --learning-rate=2.0 --hidden-size=128 --delta=8e-5 --batch-size 400 --n-layers=1 --sigma=1.0 --max-per-sample-grad-norm=1.5 --data-root="runs/charlstm/data/data/names/" --device cuda --test-every 5
117-
python -c "import torch; accuracy = torch.load('run_results_chr_lstm_classification.pt'); exit(0) if (accuracy>0.60 and accuracy<0.80) else exit(1)"
118-
119-
- name: Store test results
120-
uses: actions/upload-artifact@v2
121-
with:
122-
name: charlstm-gpu-reports
123-
path: runs/charlstm/test-reports
124-
125-
micro_benchmarks_py39_torch_release_cuda:
126-
runs-on: ubuntu-latest
127-
needs: [integrationtest_py39_torch_release_cuda]
128-
container:
129-
# https://hub.docker.com/r/nvidia/cuda
130-
image: nvidia/cuda:12.3.1-base-ubuntu22.04
131-
options: --gpus all
132-
env:
133-
TZ: 'UTC'
134-
steps:
135-
- name: Checkout
136-
uses: actions/checkout@v2
137-
138-
- name: Set up Python
139-
uses: actions/setup-python@v2
140-
with:
141-
python-version: 3.9
142-
143-
- name: Install dependencies
144-
run: |
145-
python -m pip install --upgrade pip
146-
pip install pytest coverage coveralls
147-
./scripts/install_via_pip.sh
148-
149-
- name: Install CUDA toolkit and cuDNN
150-
run: |
151-
apt-get update
152-
apt-get install -y --no-install-recommends \
153-
cuda-toolkit-11-1 \
154-
libcudnn8=8.1.1.33-1+cuda11.1 \
155-
libcudnn8-dev=8.1.1.33-1+cuda11.1
156-
157-
- name: Run benchmark integration tests (CUDA)
158-
run: |
159-
mkdir -p benchmarks/results/raw
160-
python benchmarks/run_benchmarks.py --batch_size 16 --layers "groupnorm instancenorm layernorm" --config_file ./benchmarks/config.json --root ./benchmarks/results/raw/ --cont
161-
IFS=$' ';layers=("groupnorm" "instancenorm" "layernorm"); rm -rf /tmp/report_layers; mkdir -p /tmp/report_layers; IFS=$'\n'; files=`( echo "${layers[*]}" ) | sed 's/.*/.\/benchmarks\/results\/raw\/&*/'`
162-
cp -v ${files[@]} /tmp/report_layers
163-
report_id=`IFS=$'-'; echo "${layers[*]}"`
164-
python benchmarks/generate_report.py --path-to-results /tmp/report_layers --save-path benchmarks/results/report-${report_id}.csv --format csv
165-
python benchmarks/generate_report.py --path-to-results /tmp/report_layers --save-path benchmarks/results/report-${report_id}.pkl --format pkl
166-
python benchmarks/check_threshold.py --report-path "./benchmarks/results/report-"$report_id".pkl" --metric runtime --threshold 3.0 --column "hooks/baseline"
167-
python benchmarks/check_threshold.py --report-path "./benchmarks/results/report-"$report_id".pkl" --metric memory --threshold 1.6 --column "hooks/baseline"
168-
169-
- name: Store artifacts
170-
uses: actions/upload-artifact@v2
171-
with:
172-
name: benchmarks-reports
173-
path: benchmarks/results/
97+
# To save resouces, there is no need to run all the tests.
98+
# - name: Run IMDb integration test (CUDA)
99+
# run: |
100+
# mkdir -p runs/imdb/data
101+
# mkdir -p runs/imdb/test-reports
102+
# pip install --user datasets transformers
103+
# python examples/imdb.py --lr 0.02 --sigma 1.0 -c 1.0 --batch-size 64 --max-sequence-length 256 --epochs 2 --data-root runs/imdb/data --device cuda
104+
# python -c "import torch; accuracy = torch.load('run_results_imdb_classification.pt'); exit(0) if (accuracy>0.54 and accuracy<0.66) else exit(1)"
105+
106+
# - name: Store IMDb test results
107+
# uses: actions/upload-artifact@v2
108+
# with:
109+
# name: imdb-gpu-reports
110+
# path: runs/imdb/test-reports
111+
112+
# - name: Run charlstm integration test (CUDA)
113+
# run: |
114+
# mkdir -p runs/charlstm/data
115+
# wget https://download.pytorch.org/tutorial/data.zip -O runs/charlstm/data/data.zip
116+
# unzip runs/charlstm/data/data.zip -d runs/charlstm/data
117+
# rm runs/charlstm/data/data.zip
118+
# mkdir -p runs/charlstm/test-reports
119+
# pip install scikit-learn
120+
# python examples/char-lstm-classification.py --epochs=20 --learning-rate=2.0 --hidden-size=128 --delta=8e-5 --batch-size 400 --n-layers=1 --sigma=1.0 --max-per-sample-grad-norm=1.5 --data-root="runs/charlstm/data/data/names/" --device cuda --test-every 5
121+
# python -c "import torch; accuracy = torch.load('run_results_chr_lstm_classification.pt'); exit(0) if (accuracy>0.60 and accuracy<0.80) else exit(1)"
122+
123+
# - name: Store test results
124+
# uses: actions/upload-artifact@v2
125+
# with:
126+
# name: charlstm-gpu-reports
127+
# path: runs/charlstm/test-reports
128+
129+
# We will have new benchmarks for Ghost Clipping.
130+
# micro_benchmarks_py39_torch_release_cuda:
131+
# runs-on: ubuntu-latest
132+
# needs: [integrationtest_py39_torch_release_cuda]
133+
# container:
134+
# # https://hub.docker.com/r/nvidia/cuda
135+
# image: nvidia/cuda:12.3.1-base-ubuntu22.04
136+
# options: --gpus all
137+
# env:
138+
# TZ: 'UTC'
139+
# steps:
140+
# - name: Checkout
141+
# uses: actions/checkout@v2
142+
143+
# - name: Set up Python
144+
# uses: actions/setup-python@v2
145+
# with:
146+
# python-version: 3.9
147+
148+
# - name: Install dependencies
149+
# run: |
150+
# python -m pip install --upgrade pip
151+
# pip install pytest coverage coveralls
152+
# ./scripts/install_via_pip.sh
153+
154+
# - name: Install CUDA toolkit and cuDNN
155+
# run: |
156+
# apt-get update
157+
# apt-get install -y --no-install-recommends \
158+
# cuda-toolkit-11-1 \
159+
# libcudnn8=8.1.1.33-1+cuda11.1 \
160+
# libcudnn8-dev=8.1.1.33-1+cuda11.1
161+
162+
# - name: Run benchmark integration tests (CUDA)
163+
# run: |
164+
# mkdir -p benchmarks/results/raw
165+
# python benchmarks/run_benchmarks.py --batch_size 16 --layers "groupnorm instancenorm layernorm" --config_file ./benchmarks/config.json --root ./benchmarks/results/raw/ --cont
166+
# IFS=$' ';layers=("groupnorm" "instancenorm" "layernorm"); rm -rf /tmp/report_layers; mkdir -p /tmp/report_layers; IFS=$'\n'; files=`( echo "${layers[*]}" ) | sed 's/.*/.\/benchmarks\/results\/raw\/&*/'`
167+
# cp -v ${files[@]} /tmp/report_layers
168+
# report_id=`IFS=$'-'; echo "${layers[*]}"`
169+
# python benchmarks/generate_report.py --path-to-results /tmp/report_layers --save-path benchmarks/results/report-${report_id}.csv --format csv
170+
# python benchmarks/generate_report.py --path-to-results /tmp/report_layers --save-path benchmarks/results/report-${report_id}.pkl --format pkl
171+
# python benchmarks/check_threshold.py --report-path "./benchmarks/results/report-"$report_id".pkl" --metric runtime --threshold 3.0 --column "hooks/baseline"
172+
# python benchmarks/check_threshold.py --report-path "./benchmarks/results/report-"$report_id".pkl" --metric memory --threshold 1.6 --column "hooks/baseline"
173+
174+
# - name: Store artifacts
175+
# uses: actions/upload-artifact@v2
176+
# with:
177+
# name: benchmarks-reports
178+
# path: benchmarks/results/

‎scripts/install_via_pip.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ while getopts 'ncdv:' flag; do
3535

3636
curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
3737
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
38-
sudo apt-get update && sudo apt-get install yarn
38+
sudo apt-get -y update && sudo apt-get -y install yarn
3939

4040
# yarn needs terminal info
4141
export TERM=xterm

0 commit comments

Comments
 (0)